00000800:00000200:1.1F:1769073209.994862:0:2362:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000078a4ba6a] -> 12345-192.168.204.102@tcp (2) 00000800:00000200:0.1F:1769073210.004137:0:2353:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:0.1:1769073210.004169:0:2353:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.102@tcp ip 192.168.204.102:988 00000800:00000200:0.1:1769073210.004177:0:2353:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000443e91f1 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:1.1:1769073210.037863:0:2353:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:2.1F:1769073210.044666:0:2353:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:1.0:1769074472.106370:0:30871:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-136: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.106387:0:30871:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.108226:0:30871:0:(file.c:424:ll_file_release()) START file f5.replay-single-136:[0x200001b71:0xb2:0x0](ffff8b1004846c08), flags 100000 00000002:00100000:1.0:1769074472.108346:0:30871:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005799c00 x1855007612202368/t68719477008(68719477008) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.109000:0:30871:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.109014:0:30871:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043480 x1855007612202752/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.109049:0:30871:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30871:1855007612202752:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.109214:0:30871:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074472.144863:0:30872:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10056e37b8 (ffff8b1005511908/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.144868:0:30872:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10056e37b8 (ffff8b1005511908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.144873:0:30872:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1008c9e630 (ffff8b1007c693f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.144888:0:30872:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.144904:0:30872:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.144924:0:30872:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10056e37b8 (ffff8b1005511908/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.144928:0:30872:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1008c9e630 (ffff8b1005511908/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.144939:0:30872:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.144984:0:30872:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xb3:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.144988:0:30872:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-137: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.144995:0:30872:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.145840:0:30872:0:(file.c:424:ll_file_release()) START file f5.replay-single-137:[0x200001b71:0xb3:0x0](ffff8b1004841148), flags 100000 00000002:00100000:1.0:1769074472.145942:0:30872:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064a8700 x1855007612203008/t68719477010(68719477010) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.145968:0:30872:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.145974:0:30872:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040700 x1855007612203392/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.146007:0:30872:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30872:1855007612203392:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.146124:0:30872:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074472.218853:0:30874:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100542f600 going to sleep for 11 seconds 00000008:00010000:1.0:1769074472.307929:0:30875:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074472.307935:0:30875:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc4b0, osc lock ffff8b10030deab0, flags 0x0 00010000:00010000:1.0:1769074472.307954:0:30875:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.307967:0:30875:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.307976:0:30875:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.307982:0:30875:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.307987:0:30875:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074472.307994:0:30875:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-140: read [0, 8) 00000080:00200000:1.0:1769074472.308011:0:30875:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074472.308074:0:30875:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60d80 00000008:00010000:1.0:1769074472.308118:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60d80 00010000:00010000:1.0:1769074472.308162:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074472.308214:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006041f80 - object 0x240000400:519 offset 0<>4096 00000100:00100000:1.0:1769074472.308278:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074472.308307:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612205056:192.168.204.102@tcp:3:grep.0 00000020:00010000:1.0:1769074472.325722:0:30875:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10056e3218 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.325729:0:30875:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10056e3218 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.325735:0:30875:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc4b0 (ffff8b10056851b8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.325749:0:30875:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.325765:0:30875:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.325785:0:30875:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10056e3218 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.325789:0:30875:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc4b0 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.325807:0:30875:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.325852:0:30875:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xb6:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.325858:0:30875:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-140: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.325864:0:30875:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.326870:0:30875:0:(file.c:424:ll_file_release()) START file f5.replay-single-140:[0x200001b71:0xb6:0x0](ffff8b1004831988), flags 100000 00000002:00100000:1.0:1769074472.326982:0:30875:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064a9500 x1855007612204928/t68719477016(68719477016) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.326998:0:30875:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.327005:0:30875:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041500 x1855007612205312/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.327053:0:30875:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30875:1855007612205312:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.327162:0:30875:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074472.375386:0:30876:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02e48 (ffff8b1005685998/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.375391:0:30876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02e48 (ffff8b1005685998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.375397:0:30876:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100604b630 (ffff8b1005685e18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.375409:0:30876:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9400/0x68a7113811d7f11b lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b65a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.375435:0:30876:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9400/0x68a7113811d7f11b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b65a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.375456:0:30876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02e48 (ffff8b1005685998/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.375459:0:30876:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100604b630 (ffff8b1005685998/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.375469:0:30876:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.375512:0:30876:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xb7:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.375517:0:30876:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-141: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.375523:0:30876:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200000:1.0:1769074472.529985:0:30880:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074472.530028:0:30880:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074472.530052:0:30880:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.530070:0:30880:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.530079:0:30880:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074472.530091:0:30880:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074472.530102:0:30880:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074472.530111:0:30880:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141dfbbf8 00010000:00010000:1.0:1769074472.530116:0:30880:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.530125:0:30880:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.530144:0:30880:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074472.530160:0:30880:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-145, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1003d14480, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074472.530167:0:30880:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-145, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074472.530182:0:30880:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-145,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074472.530242:0:30880:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd400/0x68a7113811d7fbe8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30880 initiator: MDT0 00010000:00010000:1.0:1769074472.530248:0:30880:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd400/0x68a7113811d7fbe8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30880 initiator: MDT0 00000020:00100000:1.0:1769074472.530260:0:30880:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074472.530275:0:30880:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041c00 x1855007612208128/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074472.530283:0:30880:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd400/0x68a7113811d7fbe8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30880 initiator: MDT0 00000100:00100000:1.0:1769074472.530306:0:30880:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30880:1855007612208128:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074472.531878:0:30880:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074472.547968:0:30880:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10056e3e48 (ffff8b1005685878/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.547975:0:30880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10056e3e48 (ffff8b1005685878/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.547980:0:30880:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1008c9e4b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.548000:0:30880:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.548017:0:30880:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.548037:0:30880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10056e3e48 (ffff8b1005685878/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.548041:0:30880:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1008c9e4b0 (ffff8b1005685878/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.548051:0:30880:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.548094:0:30880:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xbb:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.548100:0:30880:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-145: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.548106:0:30880:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.549213:0:30880:0:(file.c:424:ll_file_release()) START file f5.replay-single-145:[0x200001b71:0xbb:0x0](ffff8b10048480c8), flags 100000 00000002:00100000:1.0:1769074472.549308:0:30880:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006041c00 x1855007612208128/t68719477026(68719477026) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.549326:0:30880:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.549343:0:30880:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043b80 x1855007612208512/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.549381:0:30880:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30880:1855007612208512:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.549511:0:30880:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000080:00200000:1.0:1769074472.626965:0:30883:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074472.627005:0:30883:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074472.627028:0:30883:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.627049:0:30883:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.627071:0:30883:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074472.627083:0:30883:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074472.627088:0:30883:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074472.627098:0:30883:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141d03bf8 00010000:00010000:1.0:1769074472.627103:0:30883:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.627113:0:30883:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.627133:0:30883:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074472.627152:0:30883:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-148, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009034900, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074472.627160:0:30883:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-148, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074472.627181:0:30883:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-148,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074472.627258:0:30883:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30883 initiator: MDT0 00010000:00010000:1.0:1769074472.627265:0:30883:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30883 initiator: MDT0 00000020:00100000:1.0:1769074472.627281:0:30883:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074472.627298:0:30883:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040380 x1855007612210176/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074472.627310:0:30883:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30883 initiator: MDT0 00000100:00100000:1.0:1769074472.627341:0:30883:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30883:1855007612210176:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074472.628152:0:30883:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074472.647504:0:30883:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.647514:0:30883:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.647519:0:30883:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100604bc30 (ffff8b1005685a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.647531:0:30883:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.647547:0:30883:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.647566:0:30883:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.647570:0:30883:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100604bc30 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.647582:0:30883:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.647632:0:30883:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xbe:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.648118:0:30883:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-148: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.648127:0:30883:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.650325:0:30883:0:(file.c:424:ll_file_release()) START file f5.replay-single-148:[0x200001b71:0xbe:0x0](ffff8b100484d348), flags 100000 00000002:00100000:1.0:1769074472.650433:0:30883:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006040380 x1855007612210176/t68719477032(68719477032) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.650450:0:30883:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.650456:0:30883:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043b80 x1855007612210560/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.650485:0:30883:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30883:1855007612210560:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.651096:0:30883:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074472.703260:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612210944:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074472.708890:0:2364:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05f00 00000020:00200000:1.0:1769074472.708929:0:2364:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074472.708945:0:2364:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05f00 00000100:00100000:1.0:1769074472.708977:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aaa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612210944:192.168.204.102@tcp:3:grep.0 00000100:00100000:1.0:1769074472.832032:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612213504:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074472.835803:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05840 00000020:00200000:1.0:1769074472.835825:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074472.835850:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05840 00000100:00100000:1.0:1769074472.835881:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612213504:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074472.973236:0:30892:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074472.973277:0:30892:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074472.973298:0:30892:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.973318:0:30892:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.973328:0:30892:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074472.973338:0:30892:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074472.973343:0:30892:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074472.973351:0:30892:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141ffbbf8 00010000:00010000:1.0:1769074472.973355:0:30892:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074472.973365:0:30892:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074472.973387:0:30892:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074472.973404:0:30892:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-157, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1003d14f00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074472.973418:0:30892:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-157, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074472.973436:0:30892:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-157,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074472.973506:0:30892:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603fe00/0x68a7113811d7fc3c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30892 initiator: MDT0 00010000:00010000:1.0:1769074472.973515:0:30892:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603fe00/0x68a7113811d7fc3c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30892 initiator: MDT0 00000020:00100000:1.0:1769074472.973530:0:30892:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074472.973547:0:30892:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041c00 x1855007612215936/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074472.973558:0:30892:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603fe00/0x68a7113811d7fc3c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30892 initiator: MDT0 00000100:00100000:1.0:1769074472.973587:0:30892:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30892:1855007612215936:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074472.980321:0:30892:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98480 going to sleep for 11 seconds 00000020:00010000:1.0:1769074472.994182:0:30892:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02b78 (ffff8b1005685518/0) at cl_lock_release():245 00000020:00010000:1.0:1769074472.994187:0:30892:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02b78 (ffff8b1005685518/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074472.994192:0:30892:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100604b630 (ffff8b1005685128/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074472.994205:0:30892:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074472.994221:0:30892:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074472.994241:0:30892:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02b78 (ffff8b1005685518/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074472.994245:0:30892:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100604b630 (ffff8b1005685518/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074472.994254:0:30892:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074472.994305:0:30892:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc7:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074472.994311:0:30892:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-157: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074472.994317:0:30892:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074472.995022:0:30892:0:(file.c:424:ll_file_release()) START file f5.replay-single-157:[0x200001b71:0xc7:0x0](ffff8b100484e3c8), flags 100000 00000002:00100000:1.0:1769074472.995114:0:30892:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006041c00 x1855007612215936/t68719477050(68719477050) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074488 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074472.995129:0:30892:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074472.995136:0:30892:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041500 x1855007612216320/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074472.995164:0:30892:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30892:1855007612216320:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074472.995482:0:30892:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074473.063935:0:30893:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e027b8 (ffff8b1005685758/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.063943:0:30893:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e027b8 (ffff8b1005685758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.063948:0:30893:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1005287bb0 (ffff8b1005685488/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.063960:0:30893:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.063977:0:30893:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.063996:0:30893:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e027b8 (ffff8b1005685758/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.063999:0:30893:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1005287bb0 (ffff8b1005685758/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.064012:0:30893:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.064056:0:30893:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc8:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.064062:0:30893:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-158: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.064067:0:30893:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074473.067378:0:30893:0:(file.c:424:ll_file_release()) START file f5.replay-single-158:[0x200001b71:0xc8:0x0](ffff8b1004849988), flags 100000 00000002:00100000:1.0:1769074473.067503:0:30893:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612df80 x1855007612216576/t68719477052(68719477052) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074473.067520:0:30893:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074473.067526:0:30893:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041f80 x1855007612216960/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074473.067556:0:30893:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30893:1855007612216960:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074473.068875:0:30893:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00010000:00010000:1.0:1769074473.186947:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30897 initiator: MDT0 00010000:00010000:1.0:1769074473.186964:0:30897:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30897 initiator: MDT0 00000020:00100000:1.0:1769074473.186981:0:30897:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074473.186998:0:30897:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8380 x1855007612219136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074473.187008:0:30897:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30897 initiator: MDT0 00000100:00100000:1.0:1769074473.187038:0:30897:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30897:1855007612219136:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074473.187294:0:30897:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074473.191540:0:30897:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30897:1855007612219136:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074473.191577:0:30897:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.191605:0:30897:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194194:0:30897:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194231:0:30897:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194247:0:30897:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194268:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194277:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.194285:0:30897:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074473.194304:0:30897:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b10064a8380 x1855007612219136/t68719477060(68719477060) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074473.194318:0:30897:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a8380 x1855007612219136/t68719477060(68719477060) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074473.194333:0:30897:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074473.194546:0:30897:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.194585:0:30897:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/204 (0 0) 00000080:00200000:1.0:1769074473.194593:0:30897:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074473.194666:0:30897:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcc:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074473.194682:0:30897:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074473.194717:0:30897:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074473.194799:0:30897:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 0 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074473.194875:0:30897:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xcc:0x0](ffff8b10048521c8): rc = 0 00000080:00200000:1.0:1769074473.194891:0:30897:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xcc:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074473.194897:0:30897:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xcc:0x0](ffff8b10048521c8) for lock 0x68a7113811d7fc5f 00000080:00010000:1.0:1769074473.194911:0:30897:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xcc:0x0](ffff8b10048521c8) 00000080:00200008:1.0:1769074473.194931:0:30897:0:(file.c:1001:ll_file_open()) START file f5.replay-single-162:[0x200001b71:0xcc:0x0](ffff8b10048521c8), flags 100400 00000002:00100000:1.0:1769074473.194954:0:30897:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a8380 x1855007612219136/t68719477060(68719477060) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074473.195291:0:30897:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc5f from it ffff8b1005725de0 00010000:00010000:1.0:1769074473.195295:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.195305:0:30897:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.195336:0:30897:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xcc:0x0](ffff8b10048521c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074473.195545:0:30897:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-162:[0x200001b71:0xcc:0x0](ffff8b10048521c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074473.195554:0:30897:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xcc:0x0](ffff8b10048521c8),name=f5.replay-single-162 00000002:00010000:1.0:1769074473.195587:0:30897:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xcc:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074473.195597:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.195610:0:30897:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.195618:0:30897:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074473.195625:0:30897:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xcc:0x0](ffff8b10048521c8) 00000080:00010000:1.0:1769074473.195629:0:30897:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xcc:0x0](ffff8b10048521c8) for lock 0x68a7113811d7fc5f 00000080:00010000:1.0:1769074473.195634:0:30897:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc5f from it ffffaa914202fcd8 00010000:00010000:1.0:1769074473.195946:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.196069:0:30897:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f000/0x68a7113811d7fc5f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f97b expref: -99 pid: 30897 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.196098:0:30897:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074473.196109:0:30897:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xcc:0x0] 00000008:00010000:1.0:1769074473.196134:0:30897:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074473.196138:0:30897:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030deab0, flags 0x400001002 00010000:00010000:1.0:1769074473.196156:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196167:0:30897:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074473.196182:0:30897:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.196207:0:30897:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074473.196211:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196219:0:30897:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.196229:0:30897:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074473.196243:0:30897:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xcc:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074473.196248:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a769918 (ffff8b1005685ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.196251:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a769918 (ffff8b1005685ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.196254:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc130 (ffff8b1005685638/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.196258:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a769918 (ffff8b1005685ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.196261:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc130 (ffff8b1005685ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.196272:0:30897:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074473.196291:0:30897:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-162:[0x200001b71:0xcc:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074473.196296:0:30897:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074473.196301:0:30897:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.196305:0:30897:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074473.196309:0:30897:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xcc:0x0] 00000008:00010000:1.0:1769074473.196316:0:30897:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074473.196318:0:30897:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030deab0, flags 0x400001002 00010000:00010000:1.0:1769074473.196325:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196335:0:30897:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.196360:0:30897:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074473.196363:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196371:0:30897:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.196380:0:30897:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074473.196385:0:30897:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xcc:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074473.196388:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.196391:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.196394:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc130 (ffff8b1007c69008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.196397:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.196399:0:30897:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc130 (ffff8b1005685638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.196405:0:30897:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.196561:0:30897:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-162: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074473.196597:0:30897:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074473.196613:0:30897:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-162: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074473.196619:0:30897:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-162: desiginated mirror: 0 00000020:00200000:1.0:1769074473.196627:0:30897:0:(cl_io.c:214:cl_io_rw_init()) header@000000009f64a411[0x4, 1, [0x200001b71:0xcc:0x0] hash] 00000020:00200000:1.0:1769074473.196629:0:30897:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074473.196632:0:30897:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcc:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074473.196679:0:30897:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074473.196684:0:30897:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074473.196700:0:30897:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074473.196710:0:30897:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074473.196713:0:30897:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030deab0, flags 0x0 00010000:00010000:1.0:1769074473.196721:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196737:0:30897:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196746:0:30897:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196754:0:30897:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.196761:0:30897:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074473.196771:0:30897:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-162: read [0, 8) 00000080:00200000:1.0:1769074473.196794:0:30897:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074473.196847:0:30897:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60cc0 00000008:00010000:1.0:1769074473.196893:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60cc0 00010000:00010000:1.0:1769074473.196916:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074473.197045:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006040700 - object 0x240000400:530 offset 0<>4096 00000100:00100000:1.0:1769074473.197101:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074473.197129:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612219264:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074473.201886:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1007c60cc0 00000020:00200000:1.0:1769074473.201908:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074473.202038:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1007c60cc0 00000100:00100000:1.0:1769074473.202072:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612219264:192.168.204.102@tcp:3:grep.0 00000100:00100000:1.0:1769074473.209291:0:30897:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30897:1855007612219520:192.168.204.102@tcp:35:grep.0 00000002:00100000:1.0:1769074473.209332:0:30897:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10064a8380 x1855007612219136/t68719477060(68719477060) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769074473.231166:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612219904:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074473.235938:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05c00 00000020:00200000:1.0:1769074473.236085:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074473.236146:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05c00 00000100:00100000:1.0:1769074473.236181:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612219904:192.168.204.102@tcp:3:grep.0 00000100:00100000:1.0:1769074473.508554:0:30906:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30906:1855007612224896:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074473.508590:0:30906:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508618:0:30906:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508627:0:30906:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508673:0:30906:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508688:0:30906:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508702:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508727:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074473.508735:0:30906:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074473.508747:0:30906:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b10064aa300 x1855007612224896/t68719477078(68719477078) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074473.508761:0:30906:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064aa300 x1855007612224896/t68719477078(68719477078) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074473.508775:0:30906:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074473.508802:0:30906:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.508814:0:30906:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/213 (0 0) 00000080:00200000:1.0:1769074473.508822:0:30906:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074473.508878:0:30906:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074473.508893:0:30906:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074473.508929:0:30906:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074473.509108:0:30906:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 0 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074473.509185:0:30906:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xd5:0x0](ffff8b1004852a08): rc = 0 00000080:00200000:1.0:1769074473.509204:0:30906:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xd5:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074473.509220:0:30906:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd5:0x0](ffff8b1004852a08) for lock 0x68a7113811d7fc9e 00000080:00010000:1.0:1769074473.509235:0:30906:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd5:0x0](ffff8b1004852a08) 00000080:00200008:1.0:1769074473.509254:0:30906:0:(file.c:1001:ll_file_open()) START file f5.replay-single-171:[0x200001b71:0xd5:0x0](ffff8b1004852a08), flags 100400 00000002:00100000:1.0:1769074473.509282:0:30906:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064aa300 x1855007612224896/t68719477078(68719477078) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074473.509336:0:30906:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc9e from it ffff8b1005725900 00010000:00010000:1.0:1769074473.509341:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.509351:0:30906:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.509380:0:30906:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xd5:0x0](ffff8b1004852a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074473.509553:0:30906:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-171:[0x200001b71:0xd5:0x0](ffff8b1004852a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074473.509563:0:30906:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd5:0x0](ffff8b1004852a08),name=f5.replay-single-171 00000002:00010000:1.0:1769074473.509586:0:30906:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xd5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074473.509597:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.509609:0:30906:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.509616:0:30906:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074473.509622:0:30906:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd5:0x0](ffff8b1004852a08) 00000080:00010000:1.0:1769074473.509626:0:30906:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd5:0x0](ffff8b1004852a08) for lock 0x68a7113811d7fc9e 00000080:00010000:1.0:1769074473.509630:0:30906:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc9e from it ffffaa914206fcd8 00010000:00010000:1.0:1769074473.509633:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074473.509814:0:30906:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055e00/0x68a7113811d7fc9e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fab6 expref: -99 pid: 30906 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074473.509833:0:30906:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074473.509843:0:30906:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd5:0x0] 00000008:00010000:1.0:1769074473.509860:0:30906:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074473.509864:0:30906:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010accfb0, osc lock ffff8b10030df430, flags 0x400001002 00010000:00010000:1.0:1769074473.509883:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.509892:0:30906:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074473.509907:0:30906:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.509919:0:30906:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074473.509923:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.509931:0:30906:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.509942:0:30906:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074473.509957:0:30906:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xd5:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074473.510068:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.510072:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.510082:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010accfb0 (ffff8b1007c69ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.510087:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.510090:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010accfb0 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.510101:0:30906:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074473.510121:0:30906:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-171:[0x200001b71:0xd5:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074473.510128:0:30906:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074473.510135:0:30906:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.510140:0:30906:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074473.510144:0:30906:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd5:0x0] 00000008:00010000:1.0:1769074473.510151:0:30906:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074473.510153:0:30906:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010accfb0, osc lock ffff8b10030df430, flags 0x400001002 00010000:00010000:1.0:1769074473.510162:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510172:0:30906:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.510181:0:30906:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074473.510184:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510192:0:30906:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074473.510201:0:30906:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074473.510205:0:30906:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xd5:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074473.510208:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.510210:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.510219:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010accfb0 (ffff8b1005511d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.510222:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.510225:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010accfb0 (ffff8b1007c69ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.510231:0:30906:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.510236:0:30906:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-171: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074473.510268:0:30906:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074473.510283:0:30906:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-171: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074473.510289:0:30906:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-171: desiginated mirror: 0 00000020:00200000:1.0:1769074473.510297:0:30906:0:(cl_io.c:214:cl_io_rw_init()) header@000000002f8b1cb2[0x4, 1, [0x200001b71:0xd5:0x0] hash] 00000020:00200000:1.0:1769074473.510299:0:30906:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074473.510302:0:30906:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074473.510315:0:30906:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074473.510319:0:30906:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074473.510335:0:30906:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074473.510354:0:30906:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074473.510358:0:30906:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010accfb0, osc lock ffff8b10030df430, flags 0x0 00010000:00010000:1.0:1769074473.510366:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510376:0:30906:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510383:0:30906:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510390:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.510398:0:30906:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074473.510632:0:30906:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-171: read [0, 8) 00000080:00200000:1.0:1769074473.510742:0:30906:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074473.510811:0:30906:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60cc0 00000008:00010000:1.0:1769074473.510862:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60cc0 00010000:00010000:1.0:1769074473.510889:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074473.510933:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006040700 - object 0x280000400:535 offset 0<>4096 00000020:00010000:1.0:1769074473.515556:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.515565:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.515570:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010accfb0 (ffff8b1007c69ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.515582:0:30906:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.515599:0:30906:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.515617:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.515621:0:30906:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010accfb0 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.515635:0:30906:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.515759:0:30906:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd5:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.515764:0:30906:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-171: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.515771:0:30906:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074473.516634:0:30906:0:(file.c:424:ll_file_release()) START file f5.replay-single-171:[0x200001b71:0xd5:0x0](ffff8b1004852a08), flags 100000 00000002:00100000:1.0:1769074473.516937:0:30906:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064aa300 x1855007612224896/t68719477078(68719477078) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074473.516953:0:30906:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074473.516968:0:30906:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043480 x1855007612225280/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074473.516998:0:30906:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30906:1855007612225280:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074473.517162:0:30906:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98b40 going to sleep for 11 seconds 00000020:00010000:1.0:1769074473.556977:0:30907:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e023f8 (ffff8b10056852d8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.556984:0:30907:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e023f8 (ffff8b10056852d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.556990:0:30907:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1008c9ee30 (ffff8b1005685c68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.557003:0:30907:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.557020:0:30907:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.557041:0:30907:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e023f8 (ffff8b10056852d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.557045:0:30907:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1008c9ee30 (ffff8b10056852d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.557056:0:30907:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.557071:0:30907:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd6:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.557076:0:30907:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-172: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.557082:0:30907:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074473.557773:0:30907:0:(file.c:424:ll_file_release()) START file f5.replay-single-172:[0x200001b71:0xd6:0x0](ffff8b100483b248), flags 100000 00000002:00100000:1.0:1769074473.557880:0:30907:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612ca80 x1855007612225536/t68719477080(68719477080) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074473.557895:0:30907:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074473.557902:0:30907:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042d80 x1855007612225920/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074473.557931:0:30907:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30907:1855007612225920:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074473.558047:0:30907:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c980c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074473.719718:0:30911:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a798308 (ffff8b10055113f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.719724:0:30911:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a798308 (ffff8b10055113f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.719730:0:30911:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10052874b0 (ffff8b1007c69878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.719744:0:30911:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.719760:0:30911:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.719790:0:30911:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a798308 (ffff8b10055113f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.719794:0:30911:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10052874b0 (ffff8b10055113f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.719805:0:30911:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.719856:0:30911:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xda:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.719862:0:30911:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-176: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.719869:0:30911:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074473.720673:0:30911:0:(file.c:424:ll_file_release()) START file f5.replay-single-176:[0x200001b71:0xda:0x0](ffff8b100483cb08), flags 100000 00000002:00100000:1.0:1769074473.720766:0:30911:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612dc00 x1855007612228096/t68719477088(68719477088) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074473.720790:0:30911:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074473.720797:0:30911:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043100 x1855007612228480/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074473.720827:0:30911:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30911:1855007612228480:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074473.720929:0:30911:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074473.776593:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612228864:192.168.204.102@tcp:3:grep.0 00000020:00010000:1.0:1769074473.779834:0:30912:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02d58 (ffff8b10056857e8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.779845:0:30912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02d58 (ffff8b10056857e8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.779850:0:30912:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1008c9e4b0 (ffff8b10055117e8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.779861:0:30912:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3200/0x68a7113811d7f40f lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305be3a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.779875:0:30912:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3200/0x68a7113811d7f40f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be3a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.779893:0:30912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02d58 (ffff8b10056857e8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.779896:0:30912:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1008c9e4b0 (ffff8b10056857e8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.779906:0:30912:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.779953:0:30912:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdb:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.779958:0:30912:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-177: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.779964:0:30912:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074473.780593:0:30912:0:(file.c:424:ll_file_release()) START file f5.replay-single-177:[0x200001b71:0xdb:0x0](ffff8b100483db88), flags 100000 00000002:00100000:1.0:1769074473.782117:0:30912:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100579ad80 x1855007612228736/t68719477090(68719477090) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074473.782138:0:30912:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074473.782144:0:30912:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043800 x1855007612229120/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074473.782174:0:30912:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30912:1855007612229120:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074473.782291:0:30912:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98540 going to sleep for 11 seconds 00000020:00010000:1.0:1769074473.817746:0:30913:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02e48 (ffff8b1005685ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074473.817751:0:30913:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02e48 (ffff8b1005685ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074473.817756:0:30913:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100604b230 (ffff8b1005685758/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074473.817769:0:30913:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074473.817812:0:30913:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074473.817832:0:30913:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02e48 (ffff8b1005685ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074473.817836:0:30913:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100604b230 (ffff8b1005685ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074473.817846:0:30913:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074473.817892:0:30913:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdc:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074473.817897:0:30913:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-178: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074473.817904:0:30913:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000100:00100000:1.0:1769074473.982917:0:30915:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064ab480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30915:1855007612231040:192.168.204.102@tcp:35:grep.0 00000002:00100000:1.0:1769074473.982980:0:30915:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10064aa300 x1855007612230656/t68719477096(68719477096) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074489 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074474.006273:0:30916:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.006315:0:30916:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.006558:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.006579:0:30916:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.006589:0:30916:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.006602:0:30916:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.006607:0:30916:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.006616:0:30916:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141e33bf8 00010000:00010000:1.0:1769074474.006621:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.006686:0:30916:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.006707:0:30916:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.006725:0:30916:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-181, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009034c00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.006734:0:30916:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-181, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.006753:0:30916:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-181,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.006836:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30916 initiator: MDT0 00010000:00010000:1.0:1769074474.006845:0:30916:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30916 initiator: MDT0 00000020:00100000:1.0:1769074474.006860:0:30916:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.006876:0:30916:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042a00 x1855007612231296/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.006887:0:30916:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30916 initiator: MDT0 00000100:00100000:1.0:1769074474.006917:0:30916:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30916:1855007612231296:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.010051:0:30916:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.020948:0:30916:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30916:1855007612231296:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074474.020999:0:30916:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021027:0:30916:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021058:0:30916:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021072:0:30916:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021084:0:30916:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021099:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021108:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.021116:0:30916:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074474.021129:0:30916:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006042a00 x1855007612231296/t68719477098(68719477098) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074474.021143:0:30916:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006042a00 x1855007612231296/t68719477098(68719477098) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074474.021158:0:30916:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074474.021172:0:30916:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.021195:0:30916:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/223 (0 0) 00000080:00200000:1.0:1769074474.021203:0:30916:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074474.021259:0:30916:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.021276:0:30916:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.021316:0:30916:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074474.021404:0:30916:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 0 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.021480:0:30916:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xdf:0x0](ffff8b1004850908): rc = 0 00000080:00200000:1.0:1769074474.021497:0:30916:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xdf:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074474.021503:0:30916:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) for lock 0x68a7113811d7fce4 00000080:00010000:1.0:1769074474.021517:0:30916:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) 00000080:00200008:1.0:1769074474.021538:0:30916:0:(file.c:1001:ll_file_open()) START file f5.replay-single-181:[0x200001b71:0xdf:0x0](ffff8b1004850908), flags 100400 00000002:00100000:1.0:1769074474.021562:0:30916:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006042a00 x1855007612231296/t68719477098(68719477098) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074474.021614:0:30916:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fce4 from it ffff8b1007eb4240 00010000:00010000:1.0:1769074474.021619:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.021628:0:30916:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.021734:0:30916:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xdf:0x0](ffff8b1004850908), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074474.021765:0:30916:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-181:[0x200001b71:0xdf:0x0](ffff8b1004850908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074474.021783:0:30916:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xdf:0x0](ffff8b1004850908),name=f5.replay-single-181 00000002:00010000:1.0:1769074474.021811:0:30916:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xdf:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.021821:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.021833:0:30916:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.021840:0:30916:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.021845:0:30916:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) 00000080:00010000:1.0:1769074474.021849:0:30916:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) for lock 0x68a7113811d7fce4 00000080:00010000:1.0:1769074474.021852:0:30916:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fce4 from it ffffaa9141e33cd8 00010000:00010000:1.0:1769074474.021855:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.021863:0:30916:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.021882:0:30916:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.021890:0:30916:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xdf:0x0] 00000008:00010000:1.0:1769074474.021907:0:30916:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.021912:0:30916:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030df430, flags 0x400001002 00010000:00010000:1.0:1769074474.021929:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.021939:0:30916:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074474.021955:0:30916:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.021979:0:30916:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.021983:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.021991:0:30916:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.022001:0:30916:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.022015:0:30916:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xdf:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.022019:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a76b908 (ffff8b10056852d8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.022022:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a76b908 (ffff8b10056852d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.022025:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc130 (ffff8b1005685c68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.022030:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a76b908 (ffff8b10056852d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.022033:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc130 (ffff8b10056852d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.022041:0:30916:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074474.022061:0:30916:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-181:[0x200001b71:0xdf:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074474.022066:0:30916:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074474.022071:0:30916:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.022075:0:30916:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.022079:0:30916:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xdf:0x0] 00000008:00010000:1.0:1769074474.022086:0:30916:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.022088:0:30916:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030df430, flags 0x400001002 00010000:00010000:1.0:1769074474.022096:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022104:0:30916:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.022117:0:30916:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.022120:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022127:0:30916:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.022134:0:30916:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.022139:0:30916:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xdf:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.022141:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a76b6c0 (ffff8b1005685c68/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.022144:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a76b6c0 (ffff8b1005685c68/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.022146:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc130 (ffff8b1007c69008/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.022149:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a76b6c0 (ffff8b1005685c68/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.022152:0:30916:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc130 (ffff8b1005685c68/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.022157:0:30916:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.022162:0:30916:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-181: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074474.022196:0:30916:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074474.022212:0:30916:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-181: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074474.022218:0:30916:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-181: desiginated mirror: 0 00000020:00200000:1.0:1769074474.022226:0:30916:0:(cl_io.c:214:cl_io_rw_init()) header@0000000026a8f05b[0x4, 1, [0x200001b71:0xdf:0x0] hash] 00000020:00200000:1.0:1769074474.022228:0:30916:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074474.022231:0:30916:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.022243:0:30916:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074474.022246:0:30916:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.022261:0:30916:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074474.022269:0:30916:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074474.022272:0:30916:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1010acc130, osc lock ffff8b10030df430, flags 0x0 00010000:00010000:1.0:1769074474.022279:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022295:0:30916:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022304:0:30916:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022312:0:30916:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.022320:0:30916:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.022329:0:30916:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-181: read [0, 8) 00000080:00200000:1.0:1769074474.022343:0:30916:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074474.022396:0:30916:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60f00 00000008:00010000:1.0:1769074474.022446:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60f00 00010000:00010000:1.0:1769074474.022471:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074474.022516:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006042300 - object 0x280000400:540 offset 0<>4096 00000008:00010000:1.0:1769074474.033907:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1007c60f00 00000020:00200000:1.0:1769074474.033941:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074474.033967:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1007c60f00 00000100:00100000:1.0:1769074474.034014:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612231424:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074474.075875:0:30917:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-182, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1004dccc00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.075898:0:30917:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-182, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.075947:0:30917:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-182,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.076570:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30917 initiator: MDT0 00010000:00010000:1.0:1769074474.076583:0:30917:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30917 initiator: MDT0 00000020:00100000:1.0:1769074474.076598:0:30917:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.076614:0:30917:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042300 x1855007612231936/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.076624:0:30917:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30917 initiator: MDT0 00000100:00100000:1.0:1769074474.077082:0:30917:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30917:1855007612231936:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.077364:0:30917:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1005c98600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.091891:0:30917:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30917:1855007612231936:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074474.091929:0:30917:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.091957:0:30917:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.091978:0:30917:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.091991:0:30917:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.092005:0:30917:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.092032:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.092041:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.092049:0:30917:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074474.092061:0:30917:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006042300 x1855007612231936/t68719477100(68719477100) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074474.092075:0:30917:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006042300 x1855007612231936/t68719477100(68719477100) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074474.092089:0:30917:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074474.092103:0:30917:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.092116:0:30917:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/224 (0 0) 00000080:00200000:1.0:1769074474.092123:0:30917:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074474.092183:0:30917:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074474.092196:0:30917:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.092231:0:30917:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074474.092308:0:30917:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 0 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.092538:0:30917:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xe0:0x0](ffff8b10048542c8): rc = 0 00000080:00200000:1.0:1769074474.092560:0:30917:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xe0:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074474.092566:0:30917:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) for lock 0x68a7113811d7fceb 00000080:00010000:1.0:1769074474.092580:0:30917:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) 00000080:00200008:1.0:1769074474.092599:0:30917:0:(file.c:1001:ll_file_open()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffff8b10048542c8), flags 100400 00000002:00100000:1.0:1769074474.092624:0:30917:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006042300 x1855007612231936/t68719477100(68719477100) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074474.092872:0:30917:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fceb from it ffff8b1007eb4240 00010000:00010000:1.0:1769074474.092878:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.092887:0:30917:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.092914:0:30917:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffff8b10048542c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074474.092947:0:30917:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffff8b10048542c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074474.092954:0:30917:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffff8b10048542c8),name=f5.replay-single-182 00000002:00010000:1.0:1769074474.092988:0:30917:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.092999:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.093012:0:30917:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.093019:0:30917:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.093032:0:30917:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) 00000080:00010000:1.0:1769074474.093036:0:30917:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) for lock 0x68a7113811d7fceb 00000080:00010000:1.0:1769074474.093041:0:30917:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fceb from it ffffaa9141e3bcd8 00010000:00010000:1.0:1769074474.093044:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.093051:0:30917:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.093068:0:30917:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074474.093077:0:30917:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe0:0x0] 00000008:00010000:1.0:1769074474.093096:0:30917:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.093099:0:30917:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f24b0, osc lock ffff8b10030df7c0, flags 0x400001002 00010000:00010000:1.0:1769074474.093116:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093125:0:30917:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074474.093138:0:30917:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.093148:0:30917:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.093152:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093159:0:30917:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.093181:0:30917:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.093196:0:30917:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe0:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.093201:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.093204:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.093207:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f24b0 (ffff8b1007c69ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.093211:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.093214:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f24b0 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.093223:0:30917:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074474.093242:0:30917:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074474.093248:0:30917:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074474.093254:0:30917:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.093258:0:30917:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.093262:0:30917:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe0:0x0] 00000008:00010000:1.0:1769074474.093269:0:30917:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.093271:0:30917:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f24b0, osc lock ffff8b10030df7c0, flags 0x400001002 00010000:00010000:1.0:1769074474.093279:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093287:0:30917:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.093294:0:30917:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.093297:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093304:0:30917:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.093320:0:30917:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.093325:0:30917:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe0:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.093327:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.093330:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.093332:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f24b0 (ffff8b1005511d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.093335:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943a480 (ffff8b1007c69ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.093337:0:30917:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f24b0 (ffff8b1007c69ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.093343:0:30917:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.093348:0:30917:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-182: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074474.093379:0:30917:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074474.093716:0:30917:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-182: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074474.093724:0:30917:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-182: desiginated mirror: 0 00000020:00200000:1.0:1769074474.093733:0:30917:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c5738dd5[0x4, 1, [0x200001b71:0xe0:0x0] hash] 00000020:00200000:1.0:1769074474.093735:0:30917:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074474.093737:0:30917:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.093751:0:30917:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074474.093755:0:30917:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.093780:0:30917:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074474.093790:0:30917:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074474.093794:0:30917:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f24b0, osc lock ffff8b10030df7c0, flags 0x0 00010000:00010000:1.0:1769074474.093802:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093812:0:30917:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093821:0:30917:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093841:0:30917:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.093848:0:30917:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.093858:0:30917:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-182: read [0, 8) 00000080:00200000:1.0:1769074474.093872:0:30917:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074474.093927:0:30917:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60f00 00000008:00010000:1.0:1769074474.094046:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60f00 00010000:00010000:1.0:1769074474.094072:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074474.094117:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006041180 - object 0x240000400:540 offset 0<>4096 00000100:00100000:1.0:1769074474.094169:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074474.094194:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612232064:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074474.170956:0:30918:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.170996:0:30918:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.171020:0:30918:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.171040:0:30918:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.171049:0:30918:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.171058:0:30918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.171063:0:30918:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.171079:0:30918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420cfbf8 00010000:00010000:1.0:1769074474.171084:0:30918:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.171094:0:30918:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.171114:0:30918:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.171132:0:30918:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-183, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009035e00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.171141:0:30918:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-183, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.171162:0:30918:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-183,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.171233:0:30918:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30918 initiator: MDT0 00010000:00010000:1.0:1769074474.171240:0:30918:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30918 initiator: MDT0 00000020:00100000:1.0:1769074474.171256:0:30918:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.171272:0:30918:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042680 x1855007612232576/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.171283:0:30918:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30918 initiator: MDT0 00000100:00100000:1.0:1769074474.171309:0:30918:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30918:1855007612232576:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.171518:0:30918:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.231555:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612233344:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074474.251414:0:30920:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.251454:0:30920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.251480:0:30920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.251502:0:30920:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.251512:0:30920:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.251523:0:30920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.251527:0:30920:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.251545:0:30920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420cfbf8 00010000:00010000:1.0:1769074474.251549:0:30920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.251557:0:30920:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.251575:0:30920:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.251592:0:30920:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-185, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1014213380, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.251600:0:30920:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-185, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.251617:0:30920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-185,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.251721:0:30920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30920 initiator: MDT0 00010000:00010000:1.0:1769074474.251729:0:30920:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30920 initiator: MDT0 00000020:00100000:1.0:1769074474.251744:0:30920:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.251760:0:30920:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040e00 x1855007612233856/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.251792:0:30920:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30920 initiator: MDT0 00000100:00100000:1.0:1769074474.251820:0:30920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040e00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30920:1855007612233856:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.252183:0:30920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000020:00010000:1.0:1769074474.269849:0:30920:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.269855:0:30920:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.269860:0:30920:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10052879b0 (ffff8b1007c69908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.269872:0:30920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.269888:0:30920:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.269907:0:30920:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.269910:0:30920:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10052879b0 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.269919:0:30920:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.269933:0:30920:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe3:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.270151:0:30920:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-185: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.270159:0:30920:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.271187:0:30920:0:(file.c:424:ll_file_release()) START file f5.replay-single-185:[0x200001b71:0xe3:0x0](ffff8b1004857448), flags 100000 00000002:00100000:1.0:1769074474.271280:0:30920:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006040e00 x1855007612233856/t68719477106(68719477106) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.271295:0:30920:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.271301:0:30920:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041880 x1855007612234240/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.271341:0:30920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30920:1855007612234240:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.271952:0:30920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eca80 going to sleep for 11 seconds 00000080:00200000:1.0:1769074474.316236:0:30922:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.316274:0:30922:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.316295:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.316314:0:30922:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.316323:0:30922:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.316501:0:30922:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.316506:0:30922:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.316515:0:30922:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420d7bf8 00010000:00010000:1.0:1769074474.316520:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.316529:0:30922:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.316549:0:30922:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.316568:0:30922:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-187, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009034a80, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.316577:0:30922:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-187, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.316596:0:30922:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-187,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.318556:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30922 initiator: MDT0 00010000:00010000:1.0:1769074474.318575:0:30922:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30922 initiator: MDT0 00000020:00100000:1.0:1769074474.318615:0:30922:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.318630:0:30922:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040a80 x1855007612235136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.319521:0:30922:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30922 initiator: MDT0 00000100:00100000:1.0:1769074474.319562:0:30922:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30922:1855007612235136:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.320218:0:30922:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.338668:0:30922:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30922:1855007612235136:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074474.338701:0:30922:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338729:0:30922:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338739:0:30922:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338753:0:30922:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338796:0:30922:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338811:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338821:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.338848:0:30922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074474.338862:0:30922:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006040a80 x1855007612235136/t68719477110(68719477110) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074474.338876:0:30922:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006040a80 x1855007612235136/t68719477110(68719477110) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074474.338891:0:30922:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074474.338907:0:30922:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.338920:0:30922:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/229 (0 0) 00000080:00200000:1.0:1769074474.338928:0:30922:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074474.338982:0:30922:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074474.338995:0:30922:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.339032:0:30922:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074474.339110:0:30922:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 0 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.339181:0:30922:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xe5:0x0](ffff8b1004862a08): rc = 0 00000080:00200000:1.0:1769074474.339199:0:30922:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xe5:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074474.339205:0:30922:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) for lock 0x68a7113811d7fd0e 00000080:00010000:1.0:1769074474.339218:0:30922:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) 00000080:00200008:1.0:1769074474.339243:0:30922:0:(file.c:1001:ll_file_open()) START file f5.replay-single-187:[0x200001b71:0xe5:0x0](ffff8b1004862a08), flags 100400 00000002:00100000:1.0:1769074474.339269:0:30922:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006040a80 x1855007612235136/t68719477110(68719477110) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074474.339319:0:30922:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd0e from it ffff8b1007eb46c0 00010000:00010000:1.0:1769074474.339571:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.339582:0:30922:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.339613:0:30922:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xe5:0x0](ffff8b1004862a08), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074474.339675:0:30922:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-187:[0x200001b71:0xe5:0x0](ffff8b1004862a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074474.339683:0:30922:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe5:0x0](ffff8b1004862a08),name=f5.replay-single-187 00000002:00010000:1.0:1769074474.339708:0:30922:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.339719:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.339732:0:30922:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.339739:0:30922:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.339745:0:30922:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) 00000080:00010000:1.0:1769074474.339749:0:30922:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) for lock 0x68a7113811d7fd0e 00000080:00010000:1.0:1769074474.339753:0:30922:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd0e from it ffffaa91420d7cd8 00010000:00010000:1.0:1769074474.339756:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.339772:0:30922:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.339799:0:30922:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.339808:0:30922:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe5:0x0] 00000008:00010000:1.0:1769074474.339826:0:30922:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.339829:0:30922:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2330, osc lock ffff8b10030de4c0, flags 0x400001002 00010000:00010000:1.0:1769074474.339848:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.339857:0:30922:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074474.339870:0:30922:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.339881:0:30922:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.339885:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.339892:0:30922:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.339902:0:30922:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.339917:0:30922:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe5:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.339923:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.339926:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.339929:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2330 (ffff8b1007c69908/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.339933:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.339936:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2330 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.339949:0:30922:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074474.339969:0:30922:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-187:[0x200001b71:0xe5:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074474.339975:0:30922:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074474.339981:0:30922:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.339986:0:30922:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.339990:0:30922:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe5:0x0] 00000008:00010000:1.0:1769074474.340009:0:30922:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.340011:0:30922:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2330, osc lock ffff8b10030de4c0, flags 0x400001002 00010000:00010000:1.0:1769074474.340020:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340028:0:30922:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.340035:0:30922:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.340038:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340045:0:30922:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.340053:0:30922:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.340056:0:30922:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe5:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.340059:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.340061:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.340064:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2330 (ffff8b1005511d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.340066:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.340068:0:30922:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2330 (ffff8b1007c69908/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.340080:0:30922:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.340086:0:30922:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-187: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074474.340120:0:30922:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074474.340137:0:30922:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-187: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074474.340143:0:30922:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-187: desiginated mirror: 0 00000020:00200000:1.0:1769074474.340152:0:30922:0:(cl_io.c:214:cl_io_rw_init()) header@00000000c289ff0f[0x4, 1, [0x200001b71:0xe5:0x0] hash] 00000020:00200000:1.0:1769074474.340154:0:30922:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074474.340157:0:30922:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.340170:0:30922:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074474.340174:0:30922:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.340205:0:30922:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074474.340213:0:30922:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074474.340222:0:30922:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2330, osc lock ffff8b10030de4c0, flags 0x0 00010000:00010000:1.0:1769074474.340231:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340240:0:30922:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340248:0:30922:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340256:0:30922:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.340263:0:30922:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.340271:0:30922:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-187: read [0, 8) 00000080:00200000:1.0:1769074474.340284:0:30922:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000100:00100000:1.0:1769074474.342806:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074474.342842:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612235264:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074474.346109:0:2364:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05b40 00000020:00200000:1.0:1769074474.346132:0:2364:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074474.346154:0:2364:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05b40 00000100:00100000:1.0:1769074474.346189:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612235264:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074474.583576:0:30926:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.583613:0:30926:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.583635:0:30926:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.583677:0:30926:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.583685:0:30926:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.583696:0:30926:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.583700:0:30926:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.583709:0:30926:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420d7bf8 00010000:00010000:1.0:1769074474.583713:0:30926:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.583723:0:30926:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.584831:0:30926:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.584875:0:30926:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-191, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1014212300, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.584884:0:30926:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-191, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.584914:0:30926:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-191,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.585022:0:30926:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164ba00/0x68a7113811d7fd2a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30926 initiator: MDT0 00010000:00010000:1.0:1769074474.585033:0:30926:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164ba00/0x68a7113811d7fd2a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30926 initiator: MDT0 00000020:00100000:1.0:1769074474.585049:0:30926:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.585066:0:30926:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040000 x1855007612237696/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.585076:0:30926:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164ba00/0x68a7113811d7fd2a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30926 initiator: MDT0 00000100:00100000:1.0:1769074474.585103:0:30926:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040000 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30926:1855007612237696:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.586868:0:30926:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.615174:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612237824:192.168.204.102@tcp:3:grep.0 00000020:00010000:1.0:1769074474.623598:0:30926:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124e48 (ffff8b1007c69ea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.623603:0:30926:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124e48 (ffff8b1007c69ea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.623609:0:30926:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dc30 (ffff8b1005685ea8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.623622:0:30926:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4400/0x68a7113811d7f535 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c14a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.623845:0:30926:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4400/0x68a7113811d7f535 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c14a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.623871:0:30926:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124e48 (ffff8b1007c69ea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.623874:0:30926:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dc30 (ffff8b1007c69ea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.623884:0:30926:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.623910:0:30926:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe9:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.623928:0:30926:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-191: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.623934:0:30926:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.626027:0:30926:0:(file.c:424:ll_file_release()) START file f5.replay-single-191:[0x200001b71:0xe9:0x0](ffff8b1004825348), flags 100000 00000002:00100000:1.0:1769074474.626144:0:30926:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006040000 x1855007612237696/t68719477118(68719477118) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.626161:0:30926:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.626167:0:30926:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040a80 x1855007612238080/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.626197:0:30926:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30926:1855007612238080:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.626440:0:30926:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecd80 going to sleep for 11 seconds 00010000:00010000:1.0:1769074474.675943:0:30927:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.675970:0:30927:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.675975:0:30927:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.675984:0:30927:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.675999:0:30927:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.676009:0:30927:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xea:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.676014:0:30927:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.676017:0:30927:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.676021:0:30927:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d530 (ffff8b1007c69908/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.676025:0:30927:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438db0 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.676028:0:30927:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d530 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.676053:0:30927:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xea:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.676059:0:30927:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-192: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074474.676096:0:30927:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074474.676113:0:30927:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-192: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074474.676119:0:30927:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-192: desiginated mirror: 0 00000020:00200000:1.0:1769074474.676128:0:30927:0:(cl_io.c:214:cl_io_rw_init()) header@000000009e4c16d7[0x4, 1, [0x200001b71:0xea:0x0] hash] 00000020:00200000:1.0:1769074474.676130:0:30927:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074474.676133:0:30927:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xea:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.676149:0:30927:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074474.676153:0:30927:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.676171:0:30927:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074474.676183:0:30927:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074474.676187:0:30927:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2930, osc lock ffff8b10030df560, flags 0x0 00010000:00010000:1.0:1769074474.676197:0:30927:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.676207:0:30927:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.676216:0:30927:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.676224:0:30927:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.676231:0:30927:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.676240:0:30927:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-192: read [0, 8) 00000080:00200000:1.0:1769074474.676253:0:30927:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074474.676314:0:30927:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60300 00000008:00010000:1.0:1769074474.676804:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60300 00010000:00010000:1.0:1769074474.676838:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074474.676891:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006041880 - object 0x240000400:545 offset 0<>4096 00000008:00010000:1.0:1769074474.690278:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1007c60300 00000020:00200000:1.0:1769074474.690301:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074474.690606:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1007c60300 00000100:00100000:1.0:1769074474.690659:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612238464:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074474.714097:0:30928:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.714135:0:30928:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.714160:0:30928:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.714179:0:30928:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.714188:0:30928:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.714199:0:30928:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.714203:0:30928:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.714212:0:30928:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420dfbf8 00010000:00010000:1.0:1769074474.714217:0:30928:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.714227:0:30928:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.714247:0:30928:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.714269:0:30928:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-193, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009035800, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.714277:0:30928:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-193, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.714294:0:30928:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-193,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.714558:0:30928:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d7fd38 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30928 initiator: MDT0 00010000:00010000:1.0:1769074474.714567:0:30928:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d7fd38 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30928 initiator: MDT0 00000020:00100000:1.0:1769074474.714581:0:30928:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.714597:0:30928:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041880 x1855007612238976/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.714606:0:30928:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d7fd38 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30928 initiator: MDT0 00000100:00100000:1.0:1769074474.714633:0:30928:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30928:1855007612238976:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.715057:0:30928:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec9c0 going to sleep for 11 seconds 00000020:00010000:1.0:1769074474.743008:0:30928:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02c68 (ffff8b1005685c68/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.743013:0:30928:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02c68 (ffff8b1005685c68/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.743018:0:30928:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1005287930 (ffff8b1005685638/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.743030:0:30928:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.743046:0:30928:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.743066:0:30928:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02c68 (ffff8b1005685c68/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.743070:0:30928:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1005287930 (ffff8b1005685c68/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.743092:0:30928:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.743108:0:30928:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xeb:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.743113:0:30928:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-193: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.743120:0:30928:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.743944:0:30928:0:(file.c:424:ll_file_release()) START file f5.replay-single-193:[0x200001b71:0xeb:0x0](ffff8b100485f448), flags 100000 00000002:00100000:1.0:1769074474.744044:0:30928:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006041880 x1855007612238976/t68719477122(68719477122) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.744059:0:30928:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.744065:0:30928:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040e00 x1855007612239360/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.744094:0:30928:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040e00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30928:1855007612239360:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.744891:0:30928:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000080:00200000:1.0:1769074474.776172:0:30929:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074474.776225:0:30929:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.776249:0:30929:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.776269:0:30929:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.776277:0:30929:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.776291:0:30929:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074474.776295:0:30929:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074474.776304:0:30929:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa914206fbf8 00010000:00010000:1.0:1769074474.776309:0:30929:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074474.776328:0:30929:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074474.777093:0:30929:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074474.777116:0:30929:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-194, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1014212180, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074474.777124:0:30929:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-194, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074474.777145:0:30929:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-194,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074474.777598:0:30929:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d7fd3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30929 initiator: MDT0 00010000:00010000:1.0:1769074474.777610:0:30929:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d7fd3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30929 initiator: MDT0 00000020:00100000:1.0:1769074474.777627:0:30929:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074474.778103:0:30929:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042680 x1855007612239616/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074474.778117:0:30929:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d7fd3f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30929 initiator: MDT0 00000100:00100000:1.0:1769074474.778151:0:30929:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30929:1855007612239616:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074474.778956:0:30929:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074474.791676:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612239744:192.168.204.102@tcp:3:grep.0 00000020:00010000:1.0:1769074474.796016:0:30929:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e024e8 (ffff8b10056855a8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.796021:0:30929:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e024e8 (ffff8b10056855a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.796026:0:30929:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972de30 (ffff8b1005685f38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.796039:0:30929:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.796056:0:30929:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.796085:0:30929:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e024e8 (ffff8b10056855a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.796089:0:30929:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972de30 (ffff8b10056855a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.796099:0:30929:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.796144:0:30929:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xec:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.796149:0:30929:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-194: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.796155:0:30929:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.797060:0:30929:0:(file.c:424:ll_file_release()) START file f5.replay-single-194:[0x200001b71:0xec:0x0](ffff8b1004860908), flags 100000 00000002:00100000:1.0:1769074474.797164:0:30929:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006042680 x1855007612239616/t68719477124(68719477124) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.797180:0:30929:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.797185:0:30929:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042300 x1855007612240000/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.797214:0:30929:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30929:1855007612240000:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.797331:0:30929:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec300 going to sleep for 11 seconds 00000080:00200000:1.0:1769074474.845602:0:30930:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0xed:0x0](ffff8b10048642c8): rc = 0 00000080:00200000:1.0:1769074474.845921:0:30930:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0xed:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074474.845934:0:30930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xed:0x0](ffff8b10048642c8) for lock 0x68a7113811d7fd46 00000080:00010000:1.0:1769074474.845953:0:30930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xed:0x0](ffff8b10048642c8) 00000080:00200008:1.0:1769074474.845988:0:30930:0:(file.c:1001:ll_file_open()) START file f5.replay-single-195:[0x200001b71:0xed:0x0](ffff8b10048642c8), flags 100400 00000002:00100000:1.0:1769074474.846038:0:30930:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b100579b480 x1855007612240256/t68719477126(68719477126) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074474.846108:0:30930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd46 from it ffff8b100647b1e0 00010000:00010000:1.0:1769074474.846116:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.846152:0:30930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.846187:0:30930:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xed:0x0](ffff8b10048642c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074474.846224:0:30930:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-195:[0x200001b71:0xed:0x0](ffff8b10048642c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074474.846233:0:30930:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xed:0x0](ffff8b10048642c8),name=f5.replay-single-195 00000002:00010000:1.0:1769074474.846263:0:30930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xed:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074474.846280:0:30930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.846295:0:30930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.846304:0:30930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074474.846312:0:30930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xed:0x0](ffff8b10048642c8) 00000080:00010000:1.0:1769074474.846316:0:30930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xed:0x0](ffff8b10048642c8) for lock 0x68a7113811d7fd46 00000080:00010000:1.0:1769074474.846320:0:30930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd46 from it ffffaa9141f73cd8 00010000:00010000:1.0:1769074474.846324:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074474.846330:0:30930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164b000/0x68a7113811d7fd46 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xed:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fdfe expref: -99 pid: 30930 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074474.846353:0:30930:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xed:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074474.846367:0:30930:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xed:0x0] 00000008:00010000:1.0:1769074474.846533:0:30930:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.846538:0:30930:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2a30, osc lock ffff8b10030dfb50, flags 0x400001002 00010000:00010000:1.0:1769074474.846569:0:30930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.846581:0:30930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074474.846597:0:30930:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.846610:0:30930:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.846615:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.846624:0:30930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.846873:0:30930:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.846927:0:30930:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xed:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.846933:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.846937:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.846940:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2a30 (ffff8b1007c69908/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.846945:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.846949:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2a30 (ffff8b1005685638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.846965:0:30930:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xed:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074474.846992:0:30930:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-195:[0x200001b71:0xed:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074474.846999:0:30930:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xed:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074474.847006:0:30930:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xed:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.847020:0:30930:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xed:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074474.847025:0:30930:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xed:0x0] 00000008:00010000:1.0:1769074474.847033:0:30930:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074474.847036:0:30930:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2a30, osc lock ffff8b10030dfb50, flags 0x400001002 00010000:00010000:1.0:1769074474.847045:0:30930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847056:0:30930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.847063:0:30930:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074474.847066:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847073:0:30930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074474.847081:0:30930:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074474.847086:0:30930:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xed:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074474.847089:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.847091:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.847093:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2a30 (ffff8b1007c69ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.847095:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.847097:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2a30 (ffff8b1007c69908/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.847103:0:30930:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xed:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.847108:0:30930:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-195: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074474.847150:0:30930:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074474.847168:0:30930:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-195: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074474.847178:0:30930:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-195: desiginated mirror: 0 00000020:00200000:1.0:1769074474.847188:0:30930:0:(cl_io.c:214:cl_io_rw_init()) header@000000002885d244[0x4, 1, [0x200001b71:0xed:0x0] hash] 00000020:00200000:1.0:1769074474.847191:0:30930:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074474.847193:0:30930:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xed:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074474.847208:0:30930:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074474.847214:0:30930:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074474.847232:0:30930:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074474.847243:0:30930:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074474.847247:0:30930:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2a30, osc lock ffff8b10030dfb50, flags 0x0 00010000:00010000:1.0:1769074474.847256:0:30930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847265:0:30930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847273:0:30930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847280:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.847287:0:30930:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074474.847296:0:30930:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-195: read [0, 8) 00000080:00200000:1.0:1769074474.847312:0:30930:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074474.847377:0:30930:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60300 00000008:00010000:1.0:1769074474.847505:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60300 00010000:00010000:1.0:1769074474.847532:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074474.847596:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006041180 - object 0x280000400:547 offset 0<>4096 00000020:00010000:1.0:1769074474.856092:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.856097:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.856103:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2a30 (ffff8b1007c69908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.856115:0:30930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.856130:0:30930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.856151:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.856154:0:30930:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2a30 (ffff8b1005685638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.856164:0:30930:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.856178:0:30930:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xed:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.856183:0:30930:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-195: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.856189:0:30930:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.857675:0:30930:0:(file.c:424:ll_file_release()) START file f5.replay-single-195:[0x200001b71:0xed:0x0](ffff8b10048642c8), flags 100000 00000002:00100000:1.0:1769074474.857969:0:30930:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100579b480 x1855007612240256/t68719477126(68719477126) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.857986:0:30930:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.857993:0:30930:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042a00 x1855007612240640/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.858028:0:30930:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30930:1855007612240640:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.858229:0:30930:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000020:00010000:1.0:1769074474.965791:0:30933:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_release():245 00000020:00010000:1.0:1769074474.965803:0:30933:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074474.965809:0:30933:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1010acc130 (ffff8b1007c696c8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074474.965822:0:30933:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074474.965841:0:30933:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074474.965863:0:30933:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124d58 (ffff8b1007c69008/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074474.965867:0:30933:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1010acc130 (ffff8b1007c69008/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074474.965881:0:30933:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074474.966122:0:30933:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf0:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074474.966129:0:30933:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-198: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074474.966135:0:30933:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074474.967233:0:30933:0:(file.c:424:ll_file_release()) START file f5.replay-single-198:[0x200001b71:0xf0:0x0](ffff8b1004824b08), flags 100000 00000002:00100000:1.0:1769074474.967507:0:30933:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612d500 x1855007612242176/t68719477132(68719477132) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074490 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074474.967525:0:30933:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074474.967532:0:30933:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043800 x1855007612242560/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074474.967566:0:30933:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30933:1855007612242560:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074474.967878:0:30933:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000080:00200000:1.0:1769074475.043764:0:30936:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074475.043810:0:30936:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074475.043831:0:30936:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.043855:0:30936:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.043862:0:30936:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074475.043872:0:30936:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074475.043875:0:30936:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074475.043884:0:30936:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141ef3bf8 00010000:00010000:1.0:1769074475.043889:0:30936:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.043898:0:30936:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.043920:0:30936:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074475.043944:0:30936:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-201, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009035080, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074475.043954:0:30936:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-201, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074475.043974:0:30936:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-201,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074475.044048:0:30936:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d7fd70 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30936 initiator: MDT0 00010000:00010000:1.0:1769074475.044054:0:30936:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d7fd70 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30936 initiator: MDT0 00000020:00100000:1.0:1769074475.044066:0:30936:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074475.044079:0:30936:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043100 x1855007612244096/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074475.044087:0:30936:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d7fd70 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30936 initiator: MDT0 00000100:00100000:1.0:1769074475.044114:0:30936:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30936:1855007612244096:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074475.044327:0:30936:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec300 going to sleep for 11 seconds 00000020:00010000:1.0:1769074475.057421:0:30936:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10061244e8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074475.057431:0:30936:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10061244e8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.057437:0:30936:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1005287330 (ffff8b1007c69cf8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074475.057450:0:30936:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.057466:0:30936:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074475.057486:0:30936:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10061244e8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074475.057490:0:30936:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1005287330 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074475.057501:0:30936:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074475.057558:0:30936:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf3:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074475.057565:0:30936:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-201: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074475.057572:0:30936:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074475.058542:0:30936:0:(file.c:424:ll_file_release()) START file f5.replay-single-201:[0x200001b71:0xf3:0x0](ffff8b1004867448), flags 100000 00000002:00100000:1.0:1769074475.058951:0:30936:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006043100 x1855007612244096/t68719477138(68719477138) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074475.059036:0:30936:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074475.059043:0:30936:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042d80 x1855007612244480/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074475.059075:0:30936:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30936:1855007612244480:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074475.059262:0:30936:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ece40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074475.195137:0:30939:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000080:00200000:1.0:1769074475.249515:0:30940:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074475.249558:0:30940:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074475.249585:0:30940:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.249605:0:30940:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.249614:0:30940:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074475.249626:0:30940:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074475.249630:0:30940:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074475.249676:0:30940:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420ffbf8 00010000:00010000:1.0:1769074475.249681:0:30940:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.249691:0:30940:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.249709:0:30940:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074475.249727:0:30940:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-205, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1014212f00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074475.249735:0:30940:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-205, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074475.250630:0:30940:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-205,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074475.250785:0:30940:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d7fd8c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30940 initiator: MDT0 00010000:00010000:1.0:1769074475.250797:0:30940:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d7fd8c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30940 initiator: MDT0 00000020:00100000:1.0:1769074475.250813:0:30940:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074475.250842:0:30940:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043480 x1855007612246656/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074475.250853:0:30940:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d7fd8c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30940 initiator: MDT0 00000100:00100000:1.0:1769074475.250884:0:30940:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30940:1855007612246656:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074475.251108:0:30940:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec840 going to sleep for 11 seconds 00000020:00010000:1.0:1769074475.508160:0:30945:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f88128 (ffff8b1007e9bea8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074475.508172:0:30945:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f88128 (ffff8b1007e9bea8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.508176:0:30945:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d230 (ffff8b1007c69998/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074475.508187:0:30945:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.508202:0:30945:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074475.508220:0:30945:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f88128 (ffff8b1007e9bea8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074475.508222:0:30945:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d230 (ffff8b1007e9bea8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074475.508234:0:30945:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074475.508273:0:30945:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xfc:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074475.508278:0:30945:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-210: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074475.508283:0:30945:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1769074475.509313:0:30945:0:(file.c:424:ll_file_release()) START file f5.replay-single-210:[0x200001b71:0xfc:0x0](ffff8b100485a1c8), flags 100000 00000002:00100000:1.0:1769074475.509529:0:30945:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064abb80 x1855007612249856/t68719477156(68719477156) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074475.509543:0:30945:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074475.509549:0:30945:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040700 x1855007612250240/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074475.509586:0:30945:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30945:1855007612250240:192.168.204.102@tcp:35:grep.0 00000100:00100000:1.0:1769074475.509726:0:30945:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00010000:00010000:0.0:1769074475.646331:0:30948:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0400/0x68a7113811d7fdc4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060074 expref: -99 pid: 30948 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074475.648607:0:30948:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xff:0x0](ffff8b100486d348), inode mode 81a4 mask 44 00000080:00200008:0.0:1769074475.648673:0:30948:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-213:[0x200001b71:0xff:0x0](ffff8b100486d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074475.648684:0:30948:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xff:0x0](ffff8b100486d348),name=f5.replay-single-213 00000002:00010000:0.0:1769074475.648950:0:30948:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xff:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074475.648972:0:30948:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0400/0x68a7113811d7fdc4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xff:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060074 expref: -99 pid: 30948 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074475.648992:0:30948:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0400/0x68a7113811d7fdc4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060074 expref: -99 pid: 30948 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074475.649002:0:30948:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074475.649014:0:30948:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xff:0x0](ffff8b100486d348) 00000080:00010000:0.0:1769074475.649019:0:30948:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xff:0x0](ffff8b100486d348) for lock 0x68a7113811d7fdc4 00000080:00010000:0.0:1769074475.649027:0:30948:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdc4 from it ffffaa914210fcd8 00010000:00010000:0.0:1769074475.649031:0:30948:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0400/0x68a7113811d7fdc4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3060074 expref: -99 pid: 30948 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074475.649040:0:30948:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0400/0x68a7113811d7fdc4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xff:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060074 expref: -99 pid: 30948 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074475.649075:0:30948:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xff:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074475.649088:0:30948:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xff:0x0] 00000008:00010000:0.0:1769074475.649113:0:30948:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074475.649117:0:30948:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dbb0, osc lock ffff8b10097a2720, flags 0x400001002 00010000:00010000:0.0:1769074475.649137:0:30948:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.649174:0:30948:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074475.649191:0:30948:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074475.649204:0:30948:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074475.649208:0:30948:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.649216:0:30948:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074475.649227:0:30948:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074475.649242:0:30948:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xff:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074475.649248:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.649251:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.649254:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dbb0 (ffff8b1007c69878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.649269:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.649273:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dbb0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.649286:0:30948:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xff:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074475.649311:0:30948:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-213:[0x200001b71:0xff:0x0], ppos: 0, count: 32768 00000080:00200000:0.0:1769074475.649317:0:30948:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xff:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:0.0:1769074475.649323:0:30948:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xff:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.649328:0:30948:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xff:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074475.649339:0:30948:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xff:0x0] 00000008:00010000:0.0:1769074475.649346:0:30948:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074475.649348:0:30948:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dbb0, osc lock ffff8b10097a2720, flags 0x400001002 00010000:00010000:0.0:1769074475.649356:0:30948:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.649376:0:30948:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074475.650265:0:30948:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074475.650271:0:30948:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.650280:0:30948:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074475.650294:0:30948:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074475.650306:0:30948:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xff:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074475.650310:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.650312:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.650316:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dbb0 (ffff8b1007c69f38/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.650320:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.650323:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dbb0 (ffff8b1007c69878/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.650335:0:30948:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xff:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.650342:0:30948:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-213: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:0.0:1769074475.651991:0:30948:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:0.0:1769074475.652018:0:30948:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-213: read ppos: 0, bytes: 8 00000080:00200000:0.0:1769074475.652025:0:30948:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-213: desiginated mirror: 0 00000020:00200000:0.0:1769074475.652035:0:30948:0:(cl_io.c:214:cl_io_rw_init()) header@0000000040b33f1d[0x4, 1, [0x200001b71:0xff:0x0] hash] 00000020:00200000:0.0:1769074475.652056:0:30948:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:0.0:1769074475.652061:0:30948:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xff:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074475.652084:0:30948:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:0.0:1769074475.652089:0:30948:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074475.652114:0:30948:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:0.0:1769074475.652131:0:30948:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074475.652138:0:30948:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dbb0, osc lock ffff8b10097a2720, flags 0x0 00010000:00010000:0.0:1769074475.652164:0:30948:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.652181:0:30948:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.652195:0:30948:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.652204:0:30948:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.652212:0:30948:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074475.652221:0:30948:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-213: read [0, 8) 00000080:00200000:0.0:1769074475.652236:0:30948:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:0.0:1769074475.652333:0:30948:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1002d05480 00000008:00010000:0.0:1769074475.652379:0:18184:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1002d05480 00010000:00010000:0.0:1769074475.654012:0:18184:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074475.654072:0:18184:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b10064aa300 - object 0x280000400:556 offset 0<>4096 00000100:00100000:0.0:1769074475.654127:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074475.654173:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612251904:192.168.204.102@tcp:3:grep.0 00000008:00010000:0.0:1769074475.668549:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05480 00000020:00200000:0.0:1769074475.668566:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074475.668631:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05480 00000100:00100000:0.0:1769074475.668687:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612251904:192.168.204.102@tcp:3:grep.0 00000020:00010000:0.0:1769074475.673098:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10061245d8 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.673109:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10061245d8 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.673113:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dbb0 (ffff8b1007c69878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074475.673123:0:30948:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.673138:0:30948:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074475.673157:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10061245d8 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.673160:0:30948:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dbb0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.673171:0:30948:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:0.0:1769074475.673182:0:30948:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xff:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.673186:0:30948:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-213: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:0.0:1769074475.673192:0:30948:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:0.0:1769074475.673848:0:30948:0:(file.c:424:ll_file_release()) START file f5.replay-single-213:[0x200001b71:0xff:0x0](ffff8b100486d348), flags 100000 00000002:00100000:0.0:1769074475.673930:0:30948:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064abb80 x1855007612251776/t68719477162(68719477162) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074475.673943:0:30948:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074475.673949:0:30948:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064aa300 x1855007612252160/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074475.673975:0:30948:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30948:1855007612252160:192.168.204.102@tcp:35:grep.0 00000100:00100000:0.0:1769074475.674066:0:30948:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074475.701931:0:30948:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30948:1855007612252160:192.168.204.102@tcp:35:grep.0 00000002:00100000:1.0:1769074475.701978:0:30948:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10064abb80 x1855007612251776/t68719477162(68719477162) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00010000:0.0:1769074475.838394:0:30951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.838399:0:30951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.838403:0:30951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76e30 (ffff8b1005685a28/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074475.838414:0:30951:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.838427:0:30951:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074475.838443:0:30951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.838446:0:30951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76e30 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.838453:0:30951:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:0.0:1769074475.838465:0:30951:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x101:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.838470:0:30951:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-215: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:0.0:1769074475.838475:0:30951:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:0.0:1769074475.838895:0:30951:0:(file.c:424:ll_file_release()) START file f5.replay-single-215:[0x200001b71:0x101:0x0](ffff8b100486db88), flags 100000 00000002:00100000:0.0:1769074475.838946:0:30951:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612d880 x1855007612253056/t68719477166(68719477166) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074475.838954:0:30951:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074475.838958:0:30951:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9180 x1855007612253440/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074475.838975:0:30951:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30951:1855007612253440:192.168.204.102@tcp:35:grep.0 00000100:00100000:0.0:1769074475.839048:0:30951:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000080:00200000:1.0:1769074475.867327:0:30952:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074475.867357:0:30952:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074475.867377:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.867396:0:30952:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.867405:0:30952:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074475.867413:0:30952:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074475.867418:0:30952:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074475.867425:0:30952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420ffbf8 00010000:00010000:1.0:1769074475.867429:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.867440:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074475.867456:0:30952:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074475.867470:0:30952:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-216, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1005543980, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1769074475.867477:0:30952:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-216, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:1.0:1769074475.867497:0:30952:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-216,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1769074475.867548:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30952 initiator: MDT0 00010000:00010000:1.0:1769074475.867556:0:30952:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30952 initiator: MDT0 00000020:00100000:1.0:1769074475.867568:0:30952:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074475.867579:0:30952:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041f80 x1855007612253696/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074475.867589:0:30952:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30952 initiator: MDT0 00000100:00100000:1.0:1769074475.867609:0:30952:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30952:1855007612253696:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074475.872834:0:30952:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074475.876271:0:30952:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30952:1855007612253696:192.168.204.102@tcp:101:grep.0 00010000:00010000:1.0:1769074475.876299:0:30952:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876321:0:30952:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876330:0:30952:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876341:0:30952:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876352:0:30952:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876398:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876407:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074475.876414:0:30952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074475.876426:0:30952:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006041f80 x1855007612253696/t68719477168(68719477168) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074475.876438:0:30952:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006041f80 x1855007612253696/t68719477168(68719477168) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074475.876450:0:30952:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074475.876463:0:30952:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074475.876474:0:30952:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/258 (0 0) 00000080:00200000:1.0:1769074475.876481:0:30952:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074475.876523:0:30952:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x102:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074475.876535:0:30952:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074475.876568:0:30952:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x102:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1769074475.876633:0:30952:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 0 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074475.876733:0:30952:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x102:0x0](ffff8b100486aa08): rc = 0 00000080:00200000:1.0:1769074475.876758:0:30952:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x102:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1769074475.876764:0:30952:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x102:0x0](ffff8b100486aa08) for lock 0x68a7113811d7fdd9 00000080:00010000:1.0:1769074475.876775:0:30952:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x102:0x0](ffff8b100486aa08) 00000080:00200008:1.0:1769074475.876791:0:30952:0:(file.c:1001:ll_file_open()) START file f5.replay-single-216:[0x200001b71:0x102:0x0](ffff8b100486aa08), flags 100400 00000002:00100000:1.0:1769074475.876812:0:30952:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006041f80 x1855007612253696/t68719477168(68719477168) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074475.876861:0:30952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdd9 from it ffff8b1007eb4180 00010000:00010000:1.0:1769074475.876867:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074475.876877:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074475.879867:0:30952:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x102:0x0](ffff8b100486aa08), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074475.879905:0:30952:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-216:[0x200001b71:0x102:0x0](ffff8b100486aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074475.879913:0:30952:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x102:0x0](ffff8b100486aa08),name=f5.replay-single-216 00000002:00010000:1.0:1769074475.879941:0:30952:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x102:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074475.879959:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074475.879974:0:30952:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074475.879983:0:30952:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074475.880003:0:30952:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x102:0x0](ffff8b100486aa08) 00000080:00010000:1.0:1769074475.880007:0:30952:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x102:0x0](ffff8b100486aa08) for lock 0x68a7113811d7fdd9 00000080:00010000:1.0:1769074475.880015:0:30952:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdd9 from it ffffaa91420ffcd8 00010000:00010000:1.0:1769074475.880019:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074475.880029:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8800/0x68a7113811d7fdd9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x102:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30600dd expref: -99 pid: 30952 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074475.880052:0:30952:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x102:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074475.880063:0:30952:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x102:0x0] 00000008:00010000:1.0:1769074475.880081:0:30952:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074475.880085:0:30952:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2530, osc lock ffff8b10030dfb50, flags 0x400001002 00010000:00010000:1.0:1769074475.880112:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880124:0:30952:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074475.880138:0:30952:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074475.880150:0:30952:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074475.880155:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880164:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074475.880187:0:30952:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074475.880198:0:30952:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x102:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074475.880203:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074475.880206:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.880210:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2530 (ffff8b1007c69908/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.880214:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768920 (ffff8b1005685638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074475.880217:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2530 (ffff8b1005685638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074475.880227:0:30952:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x102:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074475.880243:0:30952:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-216:[0x200001b71:0x102:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1769074475.880248:0:30952:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x102:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074475.880254:0:30952:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x102:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074475.880260:0:30952:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x102:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074475.880264:0:30952:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x102:0x0] 00000008:00010000:1.0:1769074475.880270:0:30952:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074475.880272:0:30952:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2530, osc lock ffff8b10030dfb50, flags 0x400001002 00010000:00010000:1.0:1769074475.880280:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880289:0:30952:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074475.880298:0:30952:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074475.880301:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880310:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074475.880325:0:30952:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074475.880330:0:30952:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x102:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074475.880332:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_release():245 00000020:00010000:1.0:1769074475.880335:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.880339:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2530 (ffff8b1007c69ea8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074475.880342:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943b478 (ffff8b1007c69908/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074475.880344:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2530 (ffff8b1007c69908/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074475.880350:0:30952:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x102:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074475.880354:0:30952:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-216: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:1.0:1769074475.880383:0:30952:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1769074475.880394:0:30952:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-216: read ppos: 0, bytes: 8 00000080:00200000:1.0:1769074475.880399:0:30952:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-216: desiginated mirror: 0 00000020:00200000:1.0:1769074475.880406:0:30952:0:(cl_io.c:214:cl_io_rw_init()) header@00000000381378f5[0x4, 1, [0x200001b71:0x102:0x0] hash] 00000020:00200000:1.0:1769074475.880408:0:30952:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:1.0:1769074475.880411:0:30952:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x102:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074475.880423:0:30952:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:1.0:1769074475.880427:0:30952:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074475.880439:0:30952:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:1.0:1769074475.880448:0:30952:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074475.880450:0:30952:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2530, osc lock ffff8b10030dfb50, flags 0x0 00010000:00010000:1.0:1769074475.880459:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880467:0:30952:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.880476:0:30952:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.881067:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074475.881076:0:30952:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074475.881558:0:30952:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-216: read [0, 8) 00000080:00200000:1.0:1769074475.881585:0:30952:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:1.0:1769074475.881630:0:30952:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1007c60840 00000008:00010000:1.0:1769074475.881692:0:998:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1007c60840 00010000:00010000:1.0:1769074475.881719:0:998:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074475.881772:0:998:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006041500 - object 0x240000400:557 offset 0<>4096 00000100:00100000:1.0:1769074475.881812:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074475.881833:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612253824:192.168.204.102@tcp:3:grep.0 00000008:00010000:1.0:1769074475.890816:0:2364:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1007c60840 00000020:00200000:1.0:1769074475.890831:0:2364:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1769074475.890849:0:2364:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1007c60840 00000100:00100000:1.0:1769074475.890872:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612253824:192.168.204.102@tcp:3:grep.0 00000020:00010000:0.0:1769074475.890962:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.890967:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.890977:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2530 (ffff8b1007c69908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074475.890987:0:30952:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.891002:0:30952:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074475.891019:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02998 (ffff8b1005685638/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.891023:0:30952:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2530 (ffff8b1005685638/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.891031:0:30952:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:0.0:1769074475.891066:0:30952:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x102:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.891072:0:30952:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-216: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:0.0:1769074475.891078:0:30952:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:0.0:1769074475.891973:0:30952:0:(file.c:424:ll_file_release()) START file f5.replay-single-216:[0x200001b71:0x102:0x0](ffff8b100486aa08), flags 100000 00000002:00100000:0.0:1769074475.892043:0:30952:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006041f80 x1855007612253696/t68719477168(68719477168) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074475.892054:0:30952:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074475.892060:0:30952:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064ab800 x1855007612254080/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074475.892084:0:30952:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30952:1855007612254080:192.168.204.102@tcp:35:grep.0 00000100:00100000:0.0:1769074475.892220:0:30952:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2240 going to sleep for 11 seconds 00000100:00100000:0.0:1769074475.900042:0:30952:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30952:1855007612254080:192.168.204.102@tcp:35:grep.0 00000002:00100000:0.0:1769074475.900077:0:30952:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1006041f80 x1855007612253696/t68719477168(68719477168) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074475.904518:0:30953:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074475.904546:0:30953:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074475.904568:0:30953:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074475.904584:0:30953:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074475.904592:0:30953:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074475.904599:0:30953:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074475.904603:0:30953:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074475.904609:0:30953:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420f7bf8 00010000:00010000:0.0:1769074475.904613:0:30953:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074475.904622:0:30953:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074475.904636:0:30953:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074475.904677:0:30953:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-217, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1004dce600, open_flags 8100, mode 0 opened 0 00000080:00200000:0.0:1769074475.904684:0:30953:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-217, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:0.0:1769074475.904697:0:30953:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-217,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00000020:00010000:0.0:1769074475.974704:0:30954:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e028a8 (ffff8b1005685998/0) at cl_lock_release():245 00000020:00010000:0.0:1769074475.974710:0:30954:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e028a8 (ffff8b1005685998/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074475.974715:0:30954:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10052879b0 (ffff8b1005685e18/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074475.974726:0:30954:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074475.974750:0:30954:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074475.974779:0:30954:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e028a8 (ffff8b1005685998/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074475.974783:0:30954:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10052879b0 (ffff8b1005685998/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074475.974792:0:30954:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:0.0:1769074475.974827:0:30954:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x104:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074475.974831:0:30954:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-218: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:0.0:1769074475.974837:0:30954:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:0.0:1769074475.975358:0:30954:0:(file.c:424:ll_file_release()) START file f5.replay-single-218:[0x200001b71:0x104:0x0](ffff8b100486ec08), flags 100000 00000002:00100000:0.0:1769074475.975627:0:30954:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b100612d180 x1855007612254976/t68719477172(68719477172) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074475.975699:0:30954:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074475.975706:0:30954:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9880 x1855007612255360/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074475.975733:0:30954:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30954:1855007612255360:192.168.204.102@tcp:35:grep.0 00000100:00100000:0.0:1769074475.975868:0:30954:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2240 going to sleep for 11 seconds 00000100:00100000:0.0:1769074475.984866:0:30954:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30954:1855007612255360:192.168.204.102@tcp:35:grep.0 00000002:00100000:0.0:1769074475.984907:0:30954:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b100612d180 x1855007612254976/t68719477172(68719477172) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074475.995782:0:30955:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074475.995815:0:30955:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074475.995833:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074475.995849:0:30955:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074475.995870:0:30955:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074475.995880:0:30955:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074475.995885:0:30955:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074475.995893:0:30955:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa91420f7bf8 00010000:00010000:0.0:1769074475.995897:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074475.995907:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074475.995923:0:30955:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074475.995937:0:30955:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-219, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1004dce180, open_flags 8100, mode 0 opened 0 00000080:00200000:0.0:1769074475.995945:0:30955:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-219, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:0.0:1769074475.995959:0:30955:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-219,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:0.0:1769074475.996010:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30955 initiator: MDT0 00010000:00010000:0.0:1769074475.996018:0:30955:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30955 initiator: MDT0 00000020:00100000:0.0:1769074475.996029:0:30955:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074475.996042:0:30955:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9880 x1855007612255616/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074475.996051:0:30955:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30955 initiator: MDT0 00000100:00100000:0.0:1769074475.996076:0:30955:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30955:1855007612255616:192.168.204.102@tcp:101:grep.0 00000100:00100000:0.0:1769074475.996728:0:30955:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2240 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.008590:0:30955:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30955:1855007612255616:192.168.204.102@tcp:101:grep.0 00010000:00010000:0.0:1769074476.008615:0:30955:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008669:0:30955:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008678:0:30955:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008690:0:30955:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008702:0:30955:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008715:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008724:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.008732:0:30955:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.008752:0:30955:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b10064a9880 x1855007612255616/t68719477174(68719477174) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074476.008788:0:30955:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a9880 x1855007612255616/t68719477174(68719477174) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.008801:0:30955:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 0 00000002:00010000:0.0:1769074476.008814:0:30955:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.008825:0:30955:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/261 (0 0) 00000080:00200000:0.0:1769074476.008831:0:30955:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074476.008876:0:30955:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.008889:0:30955:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.008915:0:30955:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1769074476.008990:0:30955:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 0 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.009047:0:30955:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x105:0x0](ffff8b1004873248): rc = 0 00000080:00200000:0.0:1769074476.009062:0:30955:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x105:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:0.0:1769074476.009068:0:30955:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x105:0x0](ffff8b1004873248) for lock 0x68a7113811d7fdee 00000080:00010000:0.0:1769074476.009079:0:30955:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x105:0x0](ffff8b1004873248) 00000080:00200008:0.0:1769074476.009094:0:30955:0:(file.c:1001:ll_file_open()) START file f5.replay-single-219:[0x200001b71:0x105:0x0](ffff8b1004873248), flags 100400 00000002:00100000:0.0:1769074476.009116:0:30955:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a9880 x1855007612255616/t68719477174(68719477174) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074476.009160:0:30955:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdee from it ffff8b100610f780 00010000:00010000:0.0:1769074476.009165:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.009179:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.009202:0:30955:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x105:0x0](ffff8b1004873248), inode mode 81a4 mask 44 00000080:00200008:0.0:1769074476.009230:0:30955:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-219:[0x200001b71:0x105:0x0](ffff8b1004873248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.009236:0:30955:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x105:0x0](ffff8b1004873248),name=f5.replay-single-219 00000002:00010000:0.0:1769074476.009254:0:30955:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x105:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.009263:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.009274:0:30955:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.009281:0:30955:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.009286:0:30955:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x105:0x0](ffff8b1004873248) 00000080:00010000:0.0:1769074476.009289:0:30955:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x105:0x0](ffff8b1004873248) for lock 0x68a7113811d7fdee 00000080:00010000:0.0:1769074476.009293:0:30955:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdee from it ffffaa91420f7cd8 00010000:00010000:0.0:1769074476.009297:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.009304:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0a00/0x68a7113811d7fdee lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060146 expref: -99 pid: 30955 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.009319:0:30955:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074476.009326:0:30955:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x105:0x0] 00000008:00010000:0.0:1769074476.009339:0:30955:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.009342:0:30955:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dab0, osc lock ffff8b10097a2720, flags 0x400001002 00010000:00010000:0.0:1769074476.009362:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009372:0:30955:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074476.009385:0:30955:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.009395:0:30955:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.009398:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009406:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.009417:0:30955:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.009428:0:30955:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x105:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.009433:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.009436:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.009439:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dab0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.009442:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.009445:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dab0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.009453:0:30955:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074476.009470:0:30955:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-219:[0x200001b71:0x105:0x0], ppos: 0, count: 32768 00000080:00200000:0.0:1769074476.009475:0:30955:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:0.0:1769074476.009481:0:30955:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.009495:0:30955:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074476.009500:0:30955:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x105:0x0] 00000008:00010000:0.0:1769074476.009507:0:30955:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.009509:0:30955:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dab0, osc lock ffff8b10097a2720, flags 0x400001002 00010000:00010000:0.0:1769074476.009517:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009526:0:30955:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.009534:0:30955:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.009537:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009544:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.009553:0:30955:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.009557:0:30955:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x105:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.009560:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a769b60 (ffff8b1005685a28/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.009563:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a769b60 (ffff8b1005685a28/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.009565:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dab0 (ffff8b1007c69248/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.009568:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a769b60 (ffff8b1005685a28/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.009571:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dab0 (ffff8b1005685a28/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.009576:0:30955:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.009581:0:30955:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-219: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:0.0:1769074476.009603:0:30955:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:0.0:1769074476.009613:0:30955:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-219: read ppos: 0, bytes: 8 00000080:00200000:0.0:1769074476.009622:0:30955:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-219: desiginated mirror: 0 00000020:00200000:0.0:1769074476.009630:0:30955:0:(cl_io.c:214:cl_io_rw_init()) header@000000004639ce1f[0x4, 1, [0x200001b71:0x105:0x0] hash] 00000020:00200000:0.0:1769074476.009632:0:30955:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:0.0:1769074476.009634:0:30955:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.009676:0:30955:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:0.0:1769074476.009680:0:30955:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.009692:0:30955:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:0.0:1769074476.009699:0:30955:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074476.009702:0:30955:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dab0, osc lock ffff8b10097a2720, flags 0x0 00010000:00010000:0.0:1769074476.009709:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009718:0:30955:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009727:0:30955:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009734:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.009753:0:30955:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.009760:0:30955:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-219: read [0, 8) 00000080:00200000:0.0:1769074476.009780:0:30955:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:0.0:1769074476.009817:0:30955:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1002d05480 00000008:00010000:0.0:1769074476.009861:0:18184:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1002d05480 00010000:00010000:0.0:1769074476.009881:0:18184:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074476.009926:0:18184:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b10064a8380 - object 0x280000400:559 offset 0<>4096 00000100:00100000:0.0:1769074476.009969:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074476.009985:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612255744:192.168.204.102@tcp:3:grep.0 00000008:00010000:0.0:1769074476.014968:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05480 00000020:00200000:0.0:1769074476.014983:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074476.014997:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05480 00000100:00100000:0.0:1769074476.015021:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612255744:192.168.204.102@tcp:3:grep.0 00000020:00010000:1.0:1769074476.015180:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074476.015185:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074476.015190:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dab0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074476.015200:0:30955:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074476.015227:0:30955:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074476.015245:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02a88 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074476.015248:0:30955:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dab0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074476.015257:0:30955:0:(vvp_io.c:518:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1769074476.015267:0:30955:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074476.015272:0:30955:0:(file.c:2142:ll_file_io_generic()) f5.replay-single-219: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1769074476.015277:0:30955:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:0.0:1769074476.017626:0:30955:0:(file.c:424:ll_file_release()) START file f5.replay-single-219:[0x200001b71:0x105:0x0](ffff8b1004873248), flags 100000 00000002:00100000:0.0:1769074476.017728:0:30955:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10064a9880 x1855007612255616/t68719477174(68719477174) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074476.017751:0:30955:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074476.017757:0:30955:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8380 x1855007612256000/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074476.017781:0:30955:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30955:1855007612256000:192.168.204.102@tcp:35:grep.0 00000100:00100000:0.0:1769074476.017853:0:30955:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2a80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.024056:0:30955:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30955:1855007612256000:192.168.204.102@tcp:35:grep.0 00000002:00100000:0.0:1769074476.024091:0:30955:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10064a9880 x1855007612255616/t68719477174(68719477174) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074491 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074476.030915:0:30956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.030949:0:30956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.030968:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.030997:0:30956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.031005:0:30956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.031013:0:30956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.031017:0:30956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074476.031026:0:30956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa914211fbf8 00010000:00010000:0.0:1769074476.031031:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.031039:0:30956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.031057:0:30956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.031075:0:30956:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f5.replay-single-220, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1004dcd200, open_flags 8100, mode 0 opened 0 00000080:00200000:0.0:1769074476.031083:0:30956:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f5.replay-single-220, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:0.0:1769074476.031099:0:30956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f5.replay-single-220,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:0.0:1769074476.031154:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 30956 initiator: MDT0 00010000:00010000:0.0:1769074476.031162:0:30956:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30956 initiator: MDT0 00000020:00100000:0.0:1769074476.031173:0:30956:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074476.031187:0:30956:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8380 x1855007612256256/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074476.031197:0:30956:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 30956 initiator: MDT0 00000100:00100000:0.0:1769074476.031223:0:30956:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30956:1855007612256256:192.168.204.102@tcp:101:grep.0 00000100:00100000:0.0:1769074476.031812:0:30956:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2a80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.038320:0:30956:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:30956:1855007612256256:192.168.204.102@tcp:101:grep.0 00010000:00010000:0.0:1769074476.038351:0:30956:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038385:0:30956:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038394:0:30956:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038406:0:30956:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038418:0:30956:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038430:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038439:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.038448:0:30956:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.038460:0:30956:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b10064a8380 x1855007612256256/t68719477176(68719477176) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074492 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074476.038489:0:30956:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a8380 x1855007612256256/t68719477176(68719477176) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074492 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.038502:0:30956:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 0 00000002:00010000:0.0:1769074476.038515:0:30956:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.038525:0:30956:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/262 (0 0) 00000080:00200000:0.0:1769074476.038531:0:30956:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074476.038573:0:30956:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x106:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074476.038584:0:30956:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.038611:0:30956:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x106:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1769074476.038726:0:30956:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 0 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.038796:0:30956:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x106:0x0](ffff8b1004837448): rc = 0 00000080:00200000:0.0:1769074476.038812:0:30956:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x106:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:0.0:1769074476.038818:0:30956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x106:0x0](ffff8b1004837448) for lock 0x68a7113811d7fdf5 00000080:00010000:0.0:1769074476.038829:0:30956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x106:0x0](ffff8b1004837448) 00000080:00200008:0.0:1769074476.038845:0:30956:0:(file.c:1001:ll_file_open()) START file f5.replay-single-220:[0x200001b71:0x106:0x0](ffff8b1004837448), flags 100400 00000002:00100000:0.0:1769074476.038867:0:30956:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10064a8380 x1855007612256256/t68719477176(68719477176) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074492 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074476.038915:0:30956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdf5 from it ffff8b100610f780 00010000:00010000:0.0:1769074476.038919:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.038934:0:30956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.038957:0:30956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x106:0x0](ffff8b1004837448), inode mode 81a4 mask 44 00000080:00200008:0.0:1769074476.038984:0:30956:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-220:[0x200001b71:0x106:0x0](ffff8b1004837448), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.038990:0:30956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x106:0x0](ffff8b1004837448),name=f5.replay-single-220 00000002:00010000:0.0:1769074476.039007:0:30956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x106:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.039016:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.039026:0:30956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.039033:0:30956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.039038:0:30956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x106:0x0](ffff8b1004837448) 00000080:00010000:0.0:1769074476.039042:0:30956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x106:0x0](ffff8b1004837448) for lock 0x68a7113811d7fdf5 00000080:00010000:0.0:1769074476.039045:0:30956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdf5 from it ffffaa914211fcd8 00010000:00010000:0.0:1769074476.039048:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.039055:0:30956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0c00/0x68a7113811d7fdf5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x106:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3060169 expref: -99 pid: 30956 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.039068:0:30956:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x106:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.039075:0:30956:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x106:0x0] 00000008:00010000:0.0:1769074476.039087:0:30956:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.039091:0:30956:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d4b0, osc lock ffff8b10097a3300, flags 0x400001002 00010000:00010000:0.0:1769074476.039110:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039127:0:30956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074476.039141:0:30956:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.039151:0:30956:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.039155:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039164:0:30956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.039175:0:30956:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.039184:0:30956:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x106:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.039189:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.039192:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.039196:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d4b0 (ffff8b1007c69878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.039200:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.039204:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d4b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.039211:0:30956:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x106:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074476.039226:0:30956:0:(file.c:2474:do_file_read_iter()) START file f5.replay-single-220:[0x200001b71:0x106:0x0], ppos: 0, count: 32768 00000080:00200000:0.0:1769074476.039231:0:30956:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x106:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:0.0:1769074476.039238:0:30956:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x106:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.039251:0:30956:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x106:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074476.039255:0:30956:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x106:0x0] 00000008:00010000:0.0:1769074476.039262:0:30956:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.039264:0:30956:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d4b0, osc lock ffff8b10097a3300, flags 0x400001002 00010000:00010000:0.0:1769074476.039272:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039282:0:30956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.039292:0:30956:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.039295:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039304:0:30956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.039313:0:30956:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.039317:0:30956:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x106:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.039321:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.039324:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.039327:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d4b0 (ffff8b1007c69f38/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.039330:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009439ff0 (ffff8b1007c69878/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.039332:0:30956:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d4b0 (ffff8b1007c69878/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.039338:0:30956:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x106:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.039343:0:30956:0:(file.c:2367:file_read_confine_iter()) f5.replay-single-220: read [0, 32768] over eof, kms 8, file_size 8. 00000080:00200000:0.0:1769074476.039366:0:30956:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:0.0:1769074476.039377:0:30956:0:(file.c:1994:ll_file_io_generic()) f5.replay-single-220: read ppos: 0, bytes: 8 00000080:00200000:0.0:1769074476.039386:0:30956:0:(file.c:1759:ll_io_set_mirror()) f5.replay-single-220: desiginated mirror: 0 00000020:00200000:0.0:1769074476.039393:0:30956:0:(cl_io.c:214:cl_io_rw_init()) header@00000000b269f902[0x4, 1, [0x200001b71:0x106:0x0] hash] 00000020:00200000:0.0:1769074476.039395:0:30956:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 8) 0 0 00000080:00200000:0.0:1769074476.039397:0:30956:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x106:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.039409:0:30956:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 8) 8, 8 00020000:00200000:0.0:1769074476.039412:0:30956:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.039424:0:30956:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:0.0:1769074476.039431:0:30956:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074476.039434:0:30956:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d4b0, osc lock ffff8b10097a3300, flags 0x0 00010000:00010000:0.0:1769074476.039443:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039452:0:30956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039460:0:30956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039468:0:30956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.039475:0:30956:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.039484:0:30956:0:(vvp_io.c:827:vvp_io_read_start()) f5.replay-single-220: read [0, 8) 00000080:00200000:0.0:1769074476.039493:0:30956:0:(vvp_io.c:886:vvp_io_read_start()) count:8 iocb pos:0 00000008:00010000:0.0:1769074476.039531:0:30956:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1002d05480 00000008:00010000:0.0:1769074476.039572:0:18184:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1002d05480 00010000:00010000:0.0:1769074476.039592:0:18184:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074476.039636:0:18184:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b10064a9880 - object 0x240000400:559 offset 0<>4096 00000100:00100000:1.0:1769074476.039792:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612256384:192.168.204.102@tcp:3:grep.0 00000008:00010000:0.0:1769074476.045837:0:2365:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1002d05480 00000020:00200000:0.0:1769074476.045854:0:2365:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074476.045869:0:2365:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1002d05480 00000100:00100000:0.0:1769074476.045894:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612256384:192.168.204.102@tcp:3:grep.0 00000080:00200000:1.0:1769074476.053129:0:29778:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074476.053149:0:29778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074476.053167:0:29778:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.053180:0:29778:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074476.053188:0:29778:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074476.053194:0:29778:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074476.053198:0:29778:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074476.053204:0:29778:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141d8bbb0 00010000:00010000:1.0:1769074476.053207:0:29778:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.053215:0:29778:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074476.053246:0:29778:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074476.053255:0:29778:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 304000 00000080:00200008:1.0:1769074476.053271:0:29778:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074476.053277:0:29778:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074476.053283:0:29778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074476.053290:0:29778:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.053298:0:29778:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074476.053304:0:29778:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074476.053308:0:29778:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074476.053310:0:29778:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:1.0:1769074476.053314:0:29778:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa9141d8bcd8 00010000:00010000:1.0:1769074476.053317:0:29778:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.053325:0:29778:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074476.054502:0:29778:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) pos/size0/30208 32bit_api 0 00000002:00010000:1.0:1769074476.054524:0:29778:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: readdir flags 0 00010000:00010000:1.0:1769074476.054535:0:29778:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.054547:0:29778:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074476.055165:0:29778:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:29778:1855007612256768:192.168.204.102@tcp:37:bash.0 00000100:00100000:1.0:1769074476.055263:0:29778:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000080:00200000:0.0:1769074476.081984:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.082012:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.082032:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082048:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082055:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.082062:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.082066:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074476.082073:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.082077:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082086:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082101:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.082109:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-1, flags=8 00000080:00200008:0.0:1769074476.082116:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-1:[0x200001b71:0x2b:0x0](ffff8b101817f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.082121:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x2b:0x0](ffff8b101817f448),name=f5.replay-single-1 00000002:00010000:0.0:1769074476.082139:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.082148:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.082156:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.082163:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.082167:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x2b:0x0](ffff8b101817f448) 00000080:00010000:0.0:1769074476.082169:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x2b:0x0](ffff8b101817f448) for lock 0x68a7113811d7f7c0 00000080:00010000:0.0:1769074476.082173:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7c0 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.082175:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.082181:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.082202:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x2b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.082213:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x2b:0x0] 00000008:00010000:0.0:1769074476.082232:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.082235:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d930, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.082252:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.082263:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.082275:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.082279:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.082294:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.082305:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.082317:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x2b:0x0] updating i_size 6 i_blocks 1 00000020:00010000:0.0:1769074476.082321:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.082325:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.082327:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d930 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.082332:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.082334:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d930 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.082345:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x2b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.082604:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.082614:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.082622:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082631:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082662:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.082667:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.082669:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074476.082674:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.082677:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082685:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082699:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.082706:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-1, flags=0 00000080:00200000:0.0:1769074476.082711:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.082718:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.082724:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082732:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082747:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.082752:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.082754:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7f7b9 00000080:00010000:0.0:1769074476.082758:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7b9 from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.082761:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082768:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.082776:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.082781:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-1, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.082803:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.082951:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.082975:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x2b:0x0]: losing layout lock 00010000:00010000:0.0:1769074476.083013:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.083021:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.083027:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074476.083029:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170400/0x68a7113811d7f7c0 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e298 expref: -99 pid: 30723 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.083045:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601da00/0x68a7113811d7f7b9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf305e26e expref: -99 pid: 30264 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074476.083064:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074476.083077:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9880 x1855007612257024/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074476.083100:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257024:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.083272:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.091271:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257024:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074476.091344:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x2b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.091358:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.091394:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x2b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.091441:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x2b:0x0](ffff8b101817f448) 00000008:00010000:0.0:1769074476.091473:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 6 blocks 1 [cma]time: 1769074406 1769074406 0 by oinfo size 6 blocks 1 [cma]time 1769074406 1769074406 1769074465 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.091572:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.091590:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.092081:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.092093:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.092105:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074476.092130:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257152:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074476.092195:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.096759:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257152:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074476.096787:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096800:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096811:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096822:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096833:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096856:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096864:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.096876:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a8e00 x1855007612257152/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.096888:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096899:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074476.096906:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074476.096932:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074476.096938:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fdfc 00000080:00010000:0.0:1769074476.096943:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.096946:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdfc from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.096949:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.096957:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.097110:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.097122:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-10, flags=8 00000080:00200008:0.0:1769074476.097135:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-10:[0x200001b71:0x34:0x0](ffff8b1018171148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.097141:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x34:0x0](ffff8b1018171148),name=f5.replay-single-10 00000002:00010000:0.0:1769074476.097167:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x34:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.097177:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.097188:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.097195:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.097199:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x34:0x0](ffff8b1018171148) 00000080:00010000:0.0:1769074476.097203:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x34:0x0](ffff8b1018171148) for lock 0x68a7113811d7f7ff 00000080:00010000:0.0:1769074476.097207:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7ff from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.097210:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.097217:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.097237:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x34:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.097250:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x34:0x0] 00000008:00010000:0.0:1769074476.097269:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.097273:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972deb0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.097290:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.097300:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.097495:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.097500:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.097513:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.097524:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.097536:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x34:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074476.097540:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.097543:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.097546:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972deb0 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.097549:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.097552:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972deb0 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.097561:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x34:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.097593:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.097602:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.097609:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097617:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.097622:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.097626:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.097628:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fdfc 00000080:00010000:0.0:1769074476.097632:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdfc from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.097634:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097668:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.097683:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.097691:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-10, flags=0 00000080:00200000:0.0:1769074476.097695:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.097701:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.097707:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097714:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.097720:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.097724:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.097726:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fdfc 00000080:00010000:0.0:1769074476.097729:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fdfc from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.097733:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097749:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.097757:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.097763:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-10, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.097787:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097848:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.097872:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x34:0x0]: losing layout lock 00010000:00010000:0.0:1769074476.097905:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.097913:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.097919:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074476.097922:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170600/0x68a7113811d7f7ff lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x34:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e3b0 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.097941:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d7fdfc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306019a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074476.098069:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074476.098080:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8e00 x1855007612257408/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074476.098103:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257408:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.098246:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.105587:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257408:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074476.106278:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074476.106314:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074476.106544:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074476.106554:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074476.106565:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe03 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074476.106587:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612257536:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074476.106804:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.153884:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b100612e300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258560:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.154025:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074476.164346:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b100612e300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258560:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074476.164423:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x90:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074476.164438:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074476.164463:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x90:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074476.164520:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x90:0x0](ffff8b100482b248) 00000008:00010000:1.0:1769074476.164551:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 8 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d7ede8 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1f4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305add2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074476.164630:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074476.164710:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074476.164772:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074476.164792:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074476.164802:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074476.164821:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258688:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074476.164873:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eca80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.178963:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258688:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074476.178992:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179026:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179039:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179051:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179063:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179071:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179079:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.179091:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043b80 x1855007612258688/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.179103:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179115:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074476.179121:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074476.179149:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074476.179155:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe18 00000080:00010000:0.0:1769074476.179161:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.179170:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe18 from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.179174:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179183:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.179211:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.179224:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-103, flags=8 00000080:00200008:0.0:1769074476.179240:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-103:[0x200001b71:0x91:0x0](ffff8b100482d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.179245:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x91:0x0](ffff8b100482d348),name=f5.replay-single-103 00000002:00010000:0.0:1769074476.179277:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x91:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.179289:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.179300:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.179306:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.179312:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x91:0x0](ffff8b100482d348) 00000080:00010000:0.0:1769074476.179314:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x91:0x0](ffff8b100482d348) for lock 0x68a7113811d7fac2 00000080:00010000:0.0:1769074476.179318:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fac2 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.179321:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.179328:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.179348:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x91:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074476.179375:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x91:0x0] 00000008:00010000:0.0:1769074476.179395:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.179398:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d930, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.179415:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.179425:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.179436:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.179440:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.179448:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.179459:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.179470:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x91:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.179475:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.179478:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.179481:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d930 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.179485:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.179488:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d930 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.179498:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x91:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.179536:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.179544:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.179552:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179571:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.179578:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.179582:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.179585:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe18 00000080:00010000:0.0:1769074476.179589:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe18 from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.179592:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179600:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.179609:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.179617:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-103, flags=0 00000080:00200000:0.0:1769074476.179621:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.179627:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.179633:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179678:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.179686:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.179690:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.179693:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe18 00000080:00010000:0.0:1769074476.179697:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe18 from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.179700:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179713:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.179721:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.179727:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-103, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.179767:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179824:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.179836:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x91:0x0]: losing layout lock 00010000:00010000:0.0:1769074476.179872:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.179880:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.179887:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074476.179890:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1042169600/0x68a7113811d7fac2 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x91:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f16a expref: -99 pid: 30838 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.179911:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d7fe18 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060242 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074476.179928:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074476.179937:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064aa680 x1855007612258944/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074476.179972:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aa680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258944:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.181291:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.191109:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aa680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612258944:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074476.191172:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x91:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.191184:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.191209:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x91:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.191964:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x91:0x0](ffff8b100482d348) 00000008:00010000:0.0:1769074476.191998:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 8 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.192085:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.192105:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.192162:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.192171:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.192182:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074476.192205:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259072:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074476.192519:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.200988:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259072:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074476.201016:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201048:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201060:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201071:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201083:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201091:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201100:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.201113:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a8a80 x1855007612259072/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.201127:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201142:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074476.201148:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074476.201176:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074476.201182:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe1f 00000080:00010000:0.0:1769074476.201188:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.201197:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe1f from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.201200:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201209:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.201235:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.201312:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-104, flags=8 00000080:00200008:0.0:1769074476.201330:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-104:[0x200001b71:0x92:0x0](ffff8b10048242c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.201336:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x92:0x0](ffff8b10048242c8),name=f5.replay-single-104 00000002:00010000:0.0:1769074476.201355:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x92:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.201516:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.201528:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.201535:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.201540:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x92:0x0](ffff8b10048242c8) 00000080:00010000:0.0:1769074476.201543:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x92:0x0](ffff8b10048242c8) for lock 0x68a7113811d7fac9 00000080:00010000:0.0:1769074476.201547:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fac9 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.201550:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.201559:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.201581:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x92:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.201598:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x92:0x0] 00000008:00010000:0.0:1769074476.201617:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.201621:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972deb0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.201665:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.201676:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.201688:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.201692:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.201698:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.201708:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.201719:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x92:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.201724:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.201727:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.201730:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972deb0 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.201744:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.201747:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972deb0 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.201756:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x92:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.201791:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.201799:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.201806:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201825:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.201832:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.201836:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.201840:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe1f 00000080:00010000:0.0:1769074476.201844:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe1f from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.201847:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201855:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.201863:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.201870:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-104, flags=0 00000080:00200000:0.0:1769074476.201874:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.201880:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.201885:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201892:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.201897:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.201900:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.201903:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe1f 00000080:00010000:0.0:1769074476.201906:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe1f from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.201909:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.201922:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.201930:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.201936:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-104, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.201957:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.202011:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.202023:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x92:0x0]: losing layout lock 00010000:00010000:0.0:1769074476.202054:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.202061:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.202068:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074476.202070:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fe00/0x68a7113811d7fac9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x92:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f18d expref: -99 pid: 30839 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.202090:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe1f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306026c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074476.202108:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074476.202116:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8a80 x1855007612259328/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074476.202148:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259328:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.202693:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.208690:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259328:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074476.208768:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x92:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.208783:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.208808:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x92:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.208858:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x92:0x0](ffff8b10048242c8) 00000008:00010000:0.0:1769074476.208890:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 8 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.208993:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.209012:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.209064:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.209073:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.209082:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074476.209102:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259456:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074476.209169:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.217083:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259456:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074476.217115:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217152:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217162:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217172:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217185:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217193:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217200:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.217212:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064aad80 x1855007612259456/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.217225:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217540:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074476.217548:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074476.217578:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074476.217584:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe26 00000080:00010000:0.0:1769074476.217590:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.217600:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe26 from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.217604:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.217614:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.217675:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.217686:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-105, flags=8 00000080:00200008:0.0:1769074476.217702:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-105:[0x200001b71:0x93:0x0](ffff8b1004834b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.217708:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x93:0x0](ffff8b1004834b08),name=f5.replay-single-105 00000002:00010000:0.0:1769074476.217726:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x93:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.218209:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.218223:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.218232:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.218238:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x93:0x0](ffff8b1004834b08) 00000080:00010000:0.0:1769074476.218241:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x93:0x0](ffff8b1004834b08) for lock 0x68a7113811d7fad0 00000080:00010000:0.0:1769074476.218246:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fad0 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.218249:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.218256:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.218278:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x93:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.218538:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x93:0x0] 00000008:00010000:0.0:1769074476.218563:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.218567:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972deb0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.218671:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3be00/0x68a7113811d7ee27 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.218685:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3be00/0x68a7113811d7ee27 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.218698:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.218701:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3be00/0x68a7113811d7ee27 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ae7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.218709:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3be00/0x68a7113811d7ee27 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ae7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.218720:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.218747:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x93:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.218751:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.218754:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.218757:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972deb0 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.218761:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.218763:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972deb0 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.218774:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x93:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.218814:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.218825:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.218834:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.218855:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.218862:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.218866:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.218869:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe26 00000080:00010000:0.0:1769074476.218873:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe26 from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.218876:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.218883:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.218892:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.218899:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-105, flags=0 00000080:00200000:0.0:1769074476.218904:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.218910:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.218916:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.218924:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.218931:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.218935:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.218938:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fe26 00000080:00010000:0.0:1769074476.218942:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fe26 from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.218946:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.219953:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.224971:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.224983:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-105, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.225030:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.225108:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.225121:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x93:0x0]: losing layout lock 00010000:00010000:0.0:1769074476.225163:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.225171:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.225176:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074476.225179:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3bc00/0x68a7113811d7fad0 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x93:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f1b0 expref: -99 pid: 30840 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.225201:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d7fe26 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060296 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074476.225219:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074476.225232:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064aad80 x1855007612259712/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074476.225279:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259712:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074476.225367:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.241510:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612259712:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074476.776102:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xa8:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074476.776130:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074476.776170:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xa8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.776864:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xa8:0x0](ffff8b1004831148) 00000008:00010000:0.0:1769074476.776919:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074411 1769074411 0 by oinfo size 8 blocks 1 [cma]time 1769074411 1769074411 1769074471 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x200:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074476.777167:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.777295:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.778867:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.778887:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074476.778898:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074476.778934:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612268672:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074476.779379:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074476.784359:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064ab100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612268672:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074476.784395:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784429:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784453:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784465:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784480:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784488:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784495:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074476.784508:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064ab100 x1855007612268672/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074476.784524:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784538:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074476.784545:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074476.784585:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074476.784593:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fece 00000080:00010000:0.0:1769074476.784600:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.784610:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fece from it ffffaa914211fb58 00010000:00010000:0.0:1769074476.784614:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.784622:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.784694:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.784712:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-127, flags=8 00000080:00200008:0.0:1769074476.784739:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-127:[0x200001b71:0xa9:0x0](ffff8b1004843248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074476.784746:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xa9:0x0](ffff8b1004843248),name=f5.replay-single-127 00000002:00010000:0.0:1769074476.784772:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xa9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.784785:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.784798:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.784804:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.784809:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xa9:0x0](ffff8b1004843248) 00000080:00010000:0.0:1769074476.784811:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xa9:0x0](ffff8b1004843248) for lock 0x68a7113811d7fb6a 00000080:00010000:0.0:1769074476.784815:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fb6a from it ffffaa914211fcb8 00010000:00010000:0.0:1769074476.784817:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074476.784824:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.784853:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xa9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074476.784874:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xa9:0x0] 00000008:00010000:0.0:1769074476.784900:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074476.784905:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972deb0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074476.784932:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.784942:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.784957:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074476.784961:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074476.784968:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074476.784979:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074476.784995:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xa9:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074476.785000:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_release():245 00000020:00010000:0.0:1769074476.785003:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.785007:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972deb0 (ffff8b1007c695a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074476.785011:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100943ab58 (ffff8b1007c69f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074476.785013:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972deb0 (ffff8b1007c69f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074476.785025:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xa9:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074476.785063:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.785071:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.785078:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.785096:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.785102:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.785107:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.785109:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fece 00000080:00010000:0.0:1769074476.785113:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fece from it ffffaa914211fc10 00010000:00010000:0.0:1769074476.785116:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.785123:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.785131:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074476.785140:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-127, flags=0 00000080:00200000:0.0:1769074476.785144:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074476.785151:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074476.785156:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.785163:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.785169:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074476.785173:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074476.785175:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fece 00000080:00010000:0.0:1769074476.785179:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fece from it ffffaa914211fd68 00010000:00010000:0.0:1769074476.785182:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.785197:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074476.785206:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074476.785212:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-127, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074476.785240:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074476.785314:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074476.786050:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xa9:0x0]: losing layout lock 00010000:00010000:1.0:1769074476.792774:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074476.792797:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.792803:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074476.792807:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057cd200/0x68a7113811d7fb6a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xa9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f4b2 expref: -99 pid: 30862 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074476.792833:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d7fece lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060686 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074476.792857:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074476.792874:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040380 x1855007612268928/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074476.792905:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612268928:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074476.793092:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074476.821719:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612269056:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074476.821769:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821811:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821825:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821839:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821854:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821862:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821870:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074476.821883:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024000 x1855007612269056/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074492 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074476.821898:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d7fed5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30606b0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074476.821914:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074476.821923:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.172828:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-143, flags=0 00000080:00200000:0.0:1769074477.172836:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.172852:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.172863:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.172875:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.172881:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.172887:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.172890:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff4c 00000080:00010000:0.0:1769074477.172894:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff4c from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.172896:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.172921:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.172933:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.172940:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-143, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.173150:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.173211:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060d3c00/0x68a7113811d7fbda lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f6e2 expref: -99 pid: 30878 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.173224:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xb9:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.173263:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060d3c00/0x68a7113811d7fbda lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xb9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f6e2 expref: -99 pid: 30878 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.173271:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.173433:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.173436:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060d3c00/0x68a7113811d7fbda lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xb9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f6e2 expref: -99 pid: 30878 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.173458:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168a00/0x68a7113811d7ff4c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306097a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.173477:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.173489:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064ab100 x1855007612275840/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.173524:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612275840:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.173751:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000080:00200000:0.0:1769074477.258520:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-148, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.258561:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d7ff6f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060a4c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.258607:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xbe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f791 expref: -99 pid: 30883 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.258614:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xbe:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.258702:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xbe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f791 expref: -99 pid: 30883 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.258708:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d7ff6f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060a4c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.258714:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.258725:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603f400/0x68a7113811d7fbfd lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xbe:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f791 expref: -99 pid: 30883 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.258824:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d7ff6f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060a4c expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.258841:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.258852:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064aaa00 x1855007612277760/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.258871:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aaa00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612277760:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.259069:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2f00 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.266905:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064aaa00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612277760:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.267194:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xbe:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.267206:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.267228:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xbe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.267404:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xbe:0x0](ffff8b100484d348) 00000008:00010000:0.0:1769074477.267436:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074472 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.267514:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.267529:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.267570:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ff76 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.267576:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ff76 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.267583:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ff76 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.267597:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612277888:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.267838:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2f00 going to sleep for 11 seconds 00000080:00200000:0.0:1769074477.353178:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xc2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.353190:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc2:0x0] 00000008:00010000:0.0:1769074477.353211:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.353214:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d8b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.353231:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.353243:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.353270:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.353282:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.353291:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.353302:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.353315:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xc2:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.353319:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.353322:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.353325:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d8b0 (ffff8b1005685098/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.353329:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.353332:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d8b0 (ffff8b1007c695a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.353485:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc2:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.353524:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.353539:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.353548:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.353558:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.353565:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.353571:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.353574:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff92 00000080:00010000:0.0:1769074477.353586:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff92 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.353589:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.353597:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.353606:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.353615:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-152, flags=0 00000080:00200000:0.0:1769074477.353619:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.353626:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.353632:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.353728:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.353736:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.353821:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.353824:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff92 00000080:00010000:0.0:1769074477.353828:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff92 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.353832:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.353840:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.353850:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.353855:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-152, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.353885:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.353931:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697200/0x68a7113811d7fc19 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f81d expref: -99 pid: 30887 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.353942:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xc2:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.354178:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697200/0x68a7113811d7fc19 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f81d expref: -99 pid: 30887 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.354187:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.354195:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.354198:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697200/0x68a7113811d7fc19 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f81d expref: -99 pid: 30887 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.354218:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ff92 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b1e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.354235:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.354247:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8700 x1855007612279680/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.354271:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612279680:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.354573:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.369253:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612279808:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.369288:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369323:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369336:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369358:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369371:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369379:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369387:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.369399:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005661c00 x1855007612279808/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.369414:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369427:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.369433:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.369463:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.369471:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff99 00000080:00010000:0.0:1769074477.369476:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.369485:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff99 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.369489:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369498:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.369523:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.369534:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-153, flags=8 00000080:00200008:0.0:1769074477.369546:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-153:[0x200001b71:0xc3:0x0](ffff8b100484ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.369552:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc3:0x0](ffff8b100484ec08),name=f5.replay-single-153 00000002:00010000:0.0:1769074477.369570:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xc3:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.369581:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.369592:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.369598:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.369603:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc3:0x0](ffff8b100484ec08) 00000080:00010000:0.0:1769074477.369606:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc3:0x0](ffff8b100484ec08) for lock 0x68a7113811d7fc20 00000080:00010000:0.0:1769074477.369610:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc20 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.369613:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.369621:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.369661:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xc3:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074477.369677:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc3:0x0] 00000008:00010000:0.0:1769074477.369696:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.369699:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d8b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.369724:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.369733:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.369745:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.369749:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.369756:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.369766:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.369777:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xc3:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.369781:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.369785:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.369788:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d8b0 (ffff8b1005685098/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.369792:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.369795:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d8b0 (ffff8b1007c695a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.369805:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc3:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.369842:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.369850:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.369857:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369876:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.369896:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.369901:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.369904:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff99 00000080:00010000:0.0:1769074477.369908:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff99 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.369912:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369918:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.369927:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.369934:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-153, flags=0 00000080:00200000:0.0:1769074477.369938:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.369944:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.369949:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369957:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.369964:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.369967:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.369970:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ff99 00000080:00010000:0.0:1769074477.369973:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ff99 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.369976:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.369988:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.369997:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.370002:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-153, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.370022:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.370067:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.370077:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xc3:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.370109:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.370116:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.370121:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.370124:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055600/0x68a7113811d7fc20 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f840 expref: -99 pid: 30888 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.370141:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7ff99 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b48 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.370157:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.370165:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9500 x1855007612280064/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.370199:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280064:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.370276:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.378305:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280064:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.378384:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xc3:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.378399:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.378426:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.378477:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc3:0x0](ffff8b100484ec08) 00000008:00010000:0.0:1769074477.378510:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074472 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.378592:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.378610:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.378702:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.378712:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.378729:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.378748:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280192:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.378802:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.385478:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280192:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.385504:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385537:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385549:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385559:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385572:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385581:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385587:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.385599:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a8380 x1855007612280192/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.385614:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385628:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.385634:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.385697:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.385703:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffa0 00000080:00010000:0.0:1769074477.385709:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.385728:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffa0 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.385732:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.385740:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.385763:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.385775:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-154, flags=8 00000080:00200008:0.0:1769074477.385788:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-154:[0x200001b71:0xc4:0x0](ffff8b1004849148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.385793:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc4:0x0](ffff8b1004849148),name=f5.replay-single-154 00000002:00010000:0.0:1769074477.385814:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xc4:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.385825:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.385835:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.385842:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.385846:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc4:0x0](ffff8b1004849148) 00000080:00010000:0.0:1769074477.385849:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc4:0x0](ffff8b1004849148) for lock 0x68a7113811d7fc27 00000080:00010000:0.0:1769074477.385852:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc27 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.385856:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.385863:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.385884:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xc4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.385901:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc4:0x0] 00000008:00010000:0.0:1769074477.385920:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.385923:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d3b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.385939:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.385949:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.385972:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.385976:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.385985:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.385994:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.386005:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xc4:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.386009:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.386012:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.386015:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d3b0 (ffff8b1005685098/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.386018:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10094396d0 (ffff8b1007c695a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.386021:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d3b0 (ffff8b1007c695a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.386031:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc4:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.386080:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.386088:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.386094:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386111:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.386117:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.386122:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.386125:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffa0 00000080:00010000:0.0:1769074477.386129:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffa0 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.386133:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386140:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.386148:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.386157:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-154, flags=0 00000080:00200000:0.0:1769074477.386161:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.386167:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.386172:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386180:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.386186:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.386189:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.386192:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffa0 00000080:00010000:0.0:1769074477.386195:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffa0 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.386198:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386210:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.386220:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.386226:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-154, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.386250:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386298:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.386309:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xc4:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.386346:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.386354:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.386361:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.386364:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005697c00/0x68a7113811d7fc27 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f863 expref: -99 pid: 30889 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.386385:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060b72 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.386404:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.386412:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8380 x1855007612280448/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.386444:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280448:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.386519:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.393701:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280448:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.393768:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xc4:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.393780:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.393803:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xc4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.393850:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc4:0x0](ffff8b1004849148) 00000008:00010000:0.0:1769074477.393880:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074472 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.393954:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.393980:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.394028:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.394036:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.394046:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7ffa7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.394071:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612280576:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.394128:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00020000:00200000:0.0:1769074477.516752:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.516782:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xca:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.516836:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xca:0x0](ffff8b1004839988) 00000008:00010000:0.0:1769074477.516867:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074473 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005727c00/0x68a7113811d7f2aa lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x211:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ba82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.516958:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.516977:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.517033:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.517042:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.517051:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.517073:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612283264:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.517222:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2f00 going to sleep for 11 seconds 00010000:00010000:0.0:1769074477.524273:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.524283:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.524289:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.524292:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffd8 00000080:00010000:0.0:1769074477.524296:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffd8 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.524298:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.524305:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.524314:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.524319:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-161, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.524358:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.524413:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded400/0x68a7113811d7fc58 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f958 expref: -99 pid: 30896 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.524425:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xcb:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.524460:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded400/0x68a7113811d7fc58 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f958 expref: -99 pid: 30896 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.524468:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.524474:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.524478:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded400/0x68a7113811d7fc58 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f958 expref: -99 pid: 30896 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.524497:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d7ffd8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060cc2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.524516:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.524526:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064aa300 x1855007612283520/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.524549:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064aa300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612283520:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.524620:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00010000:00010000:0.0:1769074477.556815:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d7f2e9 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.556829:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.556850:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d7f2e9 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bb2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.556858:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d7f2e9 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.556869:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.556881:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xcd:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.556886:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.556890:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.556893:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034effb0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.556896:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.556899:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034effb0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.556910:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcd:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.556943:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.556953:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.557052:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557152:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.557159:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.557164:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.557166:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffe6 00000080:00010000:0.0:1769074477.557170:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffe6 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.557173:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557192:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.557201:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.557208:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-163, flags=0 00000080:00200000:0.0:1769074477.557212:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.557220:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.557227:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557235:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.557241:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.557245:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.557248:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7ffe6 00000080:00010000:0.0:1769074477.557251:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7ffe6 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.557254:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557261:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.557269:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.557275:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-163, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.557298:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557355:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726200/0x68a7113811d7fc66 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f99e expref: -99 pid: 30898 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.557368:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xcd:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.557400:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726200/0x68a7113811d7fc66 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xcd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f99e expref: -99 pid: 30898 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.557408:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.557414:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.557416:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726200/0x68a7113811d7fc66 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xcd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f99e expref: -99 pid: 30898 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.557432:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d7ffe6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d16 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.557449:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.557459:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064abb80 x1855007612284288/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.557483:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064abb80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612284288:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.557565:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00010000:00010000:0.0:1769074477.593676:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.593690:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.593697:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xcf:0x0](ffff8b1004855348) 00000080:00010000:0.0:1769074477.593700:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xcf:0x0](ffff8b1004855348) for lock 0x68a7113811d7fc74 00000080:00010000:0.0:1769074477.593736:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc74 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.593739:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.593747:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.593771:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcf:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074477.593785:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xcf:0x0] 00000008:00010000:0.0:1769074477.593807:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.593811:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d3b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.593832:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.593842:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.593854:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.593858:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.593866:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.593876:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.593886:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xcf:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.593891:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.593894:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.593898:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d3b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.593908:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.593911:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d3b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.593921:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcf:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.593973:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.593984:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.593992:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594002:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.594008:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.594012:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.594015:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fff4 00000080:00010000:0.0:1769074477.594019:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fff4 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.594022:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594030:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.594039:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.594048:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-165, flags=0 00000080:00200000:0.0:1769074477.594063:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.594070:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.594075:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594083:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.594095:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.594099:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.594102:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fff4 00000080:00010000:0.0:1769074477.594106:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fff4 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.594110:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594117:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.594124:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.594130:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-165, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.594153:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594201:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.594212:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xcf:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.594249:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.594257:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.594265:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.594280:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005ded200/0x68a7113811d7fc74 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xcf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f9e4 expref: -99 pid: 30900 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.594315:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d7fff4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d6a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.594565:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.594578:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064ab480 x1855007612285056/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.594601:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285056:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.594750:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00010000:00010000:0.0:1769074477.600273:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074477.604614:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064ab480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285056:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.604723:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xcf:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.604738:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.604764:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xcf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.604814:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xcf:0x0](ffff8b1004855348) 00000008:00010000:0.0:1769074477.604846:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074473 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.604926:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.604945:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.604998:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.605007:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.605017:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.605047:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285184:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.605304:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.613599:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285184:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.613624:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613636:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613666:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613681:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613693:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613717:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613725:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.613735:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a9f80 x1855007612285184/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.613749:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613777:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.613784:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.613809:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.613815:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fffb 00000080:00010000:0.0:1769074477.613821:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.613824:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fffb from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.613827:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.613836:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.613860:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.613872:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-166, flags=8 00000080:00200008:0.0:1769074477.613886:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-166:[0x200001b71:0xd0:0x0](ffff8b10048380c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.613891:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd0:0x0](ffff8b10048380c8),name=f5.replay-single-166 00000002:00010000:0.0:1769074477.613913:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xd0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.613924:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.613936:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.613944:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.613949:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd0:0x0](ffff8b10048380c8) 00000080:00010000:0.0:1769074477.613952:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd0:0x0](ffff8b10048380c8) for lock 0x68a7113811d7fc7b 00000080:00010000:0.0:1769074477.613956:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc7b from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.613959:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.613967:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.613991:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.614003:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd0:0x0] 00000008:00010000:0.0:1769074477.614021:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.614024:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d430, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.614040:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042169000/0x68a7113811d7f328 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x214:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bbd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.614049:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042169000/0x68a7113811d7f328 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bbd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.614060:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.614064:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042169000/0x68a7113811d7f328 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bbd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.614071:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042169000/0x68a7113811d7f328 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bbd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.614080:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.614091:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xd0:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.614095:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.614098:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.614100:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d430 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.614104:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.614107:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d430 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.614116:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.614147:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.614153:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.614159:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614167:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.614173:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.614177:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.614179:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fffb 00000080:00010000:0.0:1769074477.614182:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fffb from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.614185:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614192:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.614199:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.614205:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-166, flags=0 00000080:00200000:0.0:1769074477.614209:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.614213:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.614217:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614223:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.614233:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.614237:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.614240:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d7fffb 00000080:00010000:0.0:1769074477.614244:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fffb from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.614248:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614256:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.614264:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.614269:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-166, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.614290:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614341:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.614350:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xd0:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.614387:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.614394:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.614399:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.614402:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005726400/0x68a7113811d7fc7b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xd0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa07 expref: -99 pid: 30901 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.614429:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d7fffb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060d94 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.614445:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.614455:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9f80 x1855007612285440/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.614481:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285440:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.614565:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.627586:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285440:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074477.627831:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd0:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074477.627846:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074477.627872:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.627939:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xd0:0x0](ffff8b10048380c8) 00000008:00010000:1.0:1769074477.627973:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074473 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042169000/0x68a7113811d7f328 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x214:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bbd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074477.628055:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.628072:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.628127:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.628137:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.628147:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.628168:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285568:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.631836:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.636116:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285568:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.636142:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636154:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636164:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636174:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636185:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636193:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636206:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.636217:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043b80 x1855007612285568/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.636231:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636243:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.636259:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.636287:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.636294:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80002 00000080:00010000:0.0:1769074477.636300:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.636304:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80002 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.636307:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636315:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.636350:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.636361:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-167, flags=8 00000080:00200008:0.0:1769074477.636372:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-167:[0x200001b71:0xd1:0x0](ffff8b100483a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.636377:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd1:0x0](ffff8b100483a1c8),name=f5.replay-single-167 00000002:00010000:0.0:1769074477.636395:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xd1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.636405:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.636416:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.636422:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.636427:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd1:0x0](ffff8b100483a1c8) 00000080:00010000:0.0:1769074477.636430:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd1:0x0](ffff8b100483a1c8) for lock 0x68a7113811d7fc82 00000080:00010000:0.0:1769074477.636434:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc82 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.636437:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.636444:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.636462:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd1:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074477.636473:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd1:0x0] 00000008:00010000:0.0:1769074477.636490:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.636493:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d3b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.636507:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.636514:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.636522:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.636525:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.636530:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.636536:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.636545:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xd1:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.636548:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.636549:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.636551:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d3b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.636554:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.636555:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d3b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.636562:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.636590:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.636595:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.636599:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636603:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.636607:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.636610:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.636611:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80002 00000080:00010000:0.0:1769074477.636613:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80002 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.636615:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636620:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.636624:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.636629:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-167, flags=0 00000080:00200000:0.0:1769074477.636631:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.636634:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.636671:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636680:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.636685:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.636689:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.636692:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80002 00000080:00010000:0.0:1769074477.636695:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80002 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.636698:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636705:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.636721:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.636727:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-167, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.636747:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636796:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.636806:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xd1:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.636842:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.636853:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.636859:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.636862:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005deda00/0x68a7113811d7fc82 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa2a expref: -99 pid: 30902 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.636879:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d80002 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060dbe expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.636896:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.636904:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9c00 x1855007612285824/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.636926:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285824:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.637000:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.647487:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285824:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.647559:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd1:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.647572:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.647599:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.647698:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xd1:0x0](ffff8b100483a1c8) 00000008:00010000:0.0:1769074477.647747:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074412 1769074412 0 by oinfo size 8 blocks 1 [cma]time 1769074412 1769074412 1769074473 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.647842:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.647862:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.647921:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.647930:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.647939:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.647959:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285952:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.648010:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.659550:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612285952:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.659577:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659589:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659602:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659611:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659623:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659630:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659635:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.659681:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a8000 x1855007612285952/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.659696:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659718:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.659725:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.659754:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.659761:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80009 00000080:00010000:0.0:1769074477.659766:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.659769:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80009 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.659773:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.659781:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.659806:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.659818:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-168, flags=8 00000080:00200008:0.0:1769074477.659831:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-168:[0x200001b71:0xd2:0x0](ffff8b1004854b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.659836:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd2:0x0](ffff8b1004854b08),name=f5.replay-single-168 00000002:00010000:0.0:1769074477.659857:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xd2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.659866:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.659875:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.659882:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.659886:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd2:0x0](ffff8b1004854b08) 00000080:00010000:0.0:1769074477.659889:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd2:0x0](ffff8b1004854b08) for lock 0x68a7113811d7fc89 00000080:00010000:0.0:1769074477.659893:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fc89 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.659896:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.659902:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.659924:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.659936:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd2:0x0] 00000008:00010000:0.0:1769074477.659954:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.659957:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d430, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.659974:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.659984:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.659996:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.659999:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.660007:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.660020:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.660031:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xd2:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.660035:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.660038:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.660041:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d430 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.660045:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.660048:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d430 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.660070:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd2:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.660104:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.660111:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.660118:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660124:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.660130:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.660134:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.660136:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80009 00000080:00010000:0.0:1769074477.660140:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80009 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.660143:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660149:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.660157:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.660163:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-168, flags=0 00000080:00200000:0.0:1769074477.660167:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.660173:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.660177:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660184:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.660189:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.660192:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.660195:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80009 00000080:00010000:0.0:1769074477.660197:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80009 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.660200:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660206:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.660213:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.660219:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-168, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.660240:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660283:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.660294:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xd2:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.660326:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.660338:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.660344:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.660347:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727800/0x68a7113811d7fc89 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xd2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fa4d expref: -99 pid: 30903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.660365:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80009 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060de8 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.660381:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.660389:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a8000 x1855007612286208/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.660411:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a8000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612286208:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.660477:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.667977:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a8000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612286208:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.668038:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xd2:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.668063:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.668089:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xd2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.668139:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xd2:0x0](ffff8b1004854b08) 00000008:00010000:0.0:1769074477.668174:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074473 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.668460:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.668481:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.668540:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.668549:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.668558:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.668578:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612286336:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.668702:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.676929:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612286336:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.677014:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677028:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677040:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677844:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677868:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677876:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677884:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.677899:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10064a9180 x1855007612286336/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.677912:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80010 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060e12 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.677926:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.677934:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:1.0:1769074477.697778:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612286976:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074477.697854:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x3b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074477.697869:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074477.697903:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x3b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.697954:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x3b:0x0](ffff8b1018121148) 00000008:00010000:1.0:1769074477.697994:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cada00/0x68a7113811d7e6ef lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059b3a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074477.698092:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.698110:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.698175:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8001e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.698184:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8001e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.698193:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8001e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.698215:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612287104:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.698303:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.728393:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612287872:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074477.728492:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612288128:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074477.728554:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074477.728561:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612288000:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074477.740901:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612288000:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074477.740913:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612287872:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074477.740924:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612288128:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074477.741580:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8002c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.741591:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8002c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.741601:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8002c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.741623:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612288256:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.741737:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.855194:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b100604f100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290432:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074477.855508:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074477.855522:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074477.855549:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.855601:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xdb:0x0](ffff8b100483db88) 00000008:00010000:1.0:1769074477.855634:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074473 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3200/0x68a7113811d7f40f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be3a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074477.855774:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.855792:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.855843:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.855852:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.855861:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.855880:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290560:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.855971:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.860930:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290560:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074477.860957:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.860978:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.860989:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861000:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861011:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861031:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861038:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074477.861050:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043480 x1855007612290560/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074477.861064:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861075:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074477.861082:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074477.861108:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:1.0:1769074477.861115:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80056 00000080:00010000:1.0:1769074477.861120:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.861123:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80056 from it ffffaa914211fb58 00010000:00010000:1.0:1769074477.861127:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861135:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.861159:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074477.861170:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-178, flags=8 00000080:00200008:1.0:1769074477.861183:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-178:[0x200001b71:0xdc:0x0](ffff8b1004856c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074477.861188:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xdc:0x0](ffff8b1004856c08),name=f5.replay-single-178 00000002:00010000:1.0:1769074477.861212:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xdc:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.861222:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.861233:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.861239:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.861244:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xdc:0x0](ffff8b1004856c08) 00000080:00010000:1.0:1769074477.861247:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xdc:0x0](ffff8b1004856c08) for lock 0x68a7113811d7fccf 00000080:00010000:1.0:1769074477.861251:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fccf from it ffffaa914211fcb8 00010000:00010000:1.0:1769074477.861254:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.861261:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.861291:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdc:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074477.861302:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xdc:0x0] 00000008:00010000:1.0:1769074477.861320:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074477.861324:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2cb0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074477.861351:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074477.861362:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074477.861372:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074477.861376:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074477.861392:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074477.861404:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074477.861416:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xdc:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074477.861421:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074477.861424:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074477.861427:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2cb0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074477.861430:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074477.861433:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2cb0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074477.861443:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdc:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.861477:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.861486:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.861492:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861501:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.861507:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.861510:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.861513:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80056 00000080:00010000:1.0:1769074477.861516:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80056 from it ffffaa914211fc10 00010000:00010000:1.0:1769074477.861518:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861524:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.861538:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074477.861545:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-178, flags=0 00000080:00200000:1.0:1769074477.861548:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.861554:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.861560:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861567:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.861573:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.861577:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.861579:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80056 00000080:00010000:1.0:1769074477.861583:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80056 from it ffffaa914211fd68 00010000:00010000:1.0:1769074477.861585:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861592:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.861600:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074477.861607:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-178, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074477.861628:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861704:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.861764:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xdc:0x0]: losing layout lock 00010000:00010000:1.0:1769074477.861800:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.861809:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.861816:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074477.861819:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005727e00/0x68a7113811d7fccf lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xdc:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbab expref: -99 pid: 30913 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.861836:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80056 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060fb6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074477.861853:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074477.861862:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043480 x1855007612290816/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074477.861884:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290816:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074477.861951:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.869080:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290816:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074477.869145:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdc:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074477.869159:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074477.869187:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdc:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.869244:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xdc:0x0](ffff8b1004856c08) 00000008:00010000:1.0:1769074477.869288:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074473 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074477.869406:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.869424:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.869484:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.869493:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.869503:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.869523:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290944:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.869593:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.873556:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612290944:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074477.873585:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.873597:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.873609:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.873621:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.873633:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874006:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874013:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074477.874026:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006042d80 x1855007612290944/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074477.874040:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874053:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074477.874059:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074477.874085:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:1.0:1769074477.874092:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8005d 00000080:00010000:1.0:1769074477.874098:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.874101:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8005d from it ffffaa914211fb58 00010000:00010000:1.0:1769074477.874104:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874112:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.874136:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074477.874147:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-179, flags=8 00000080:00200008:1.0:1769074477.874160:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-179:[0x200001b71:0xdd:0x0](ffff8b100485e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074477.874165:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xdd:0x0](ffff8b100485e3c8),name=f5.replay-single-179 00000002:00010000:1.0:1769074477.874193:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xdd:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.874204:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.874216:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.874223:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.874227:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xdd:0x0](ffff8b100485e3c8) 00000080:00010000:1.0:1769074477.874230:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xdd:0x0](ffff8b100485e3c8) for lock 0x68a7113811d7fcd6 00000080:00010000:1.0:1769074477.874234:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fcd6 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074477.874237:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.874244:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.874277:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdd:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074477.874291:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xdd:0x0] 00000008:00010000:1.0:1769074477.874310:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074477.874314:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2cb0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074477.874339:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074477.874349:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074477.874362:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074477.874366:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074477.874379:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074477.874390:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074477.874402:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xdd:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074477.874406:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074477.874409:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074477.874412:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2cb0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074477.874417:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074477.874420:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2cb0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074477.874430:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdd:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.874461:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.874470:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.874477:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874486:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.874493:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.874497:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.874500:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8005d 00000080:00010000:1.0:1769074477.874503:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8005d from it ffffaa914211fc10 00010000:00010000:1.0:1769074477.874506:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874513:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.874530:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074477.874537:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-179, flags=0 00000080:00200000:1.0:1769074477.874541:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.874547:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.874553:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874561:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.874568:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074477.874571:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.874573:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8005d 00000080:00010000:1.0:1769074477.874577:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8005d from it ffffaa914211fd68 00010000:00010000:1.0:1769074477.874580:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874588:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.874595:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074477.874600:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-179, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074477.874621:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874688:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074477.874720:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xdd:0x0]: losing layout lock 00010000:00010000:1.0:1769074477.874751:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.874764:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.874771:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074477.874773:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1000031400/0x68a7113811d7fcd6 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xdd:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbce expref: -99 pid: 30914 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074477.874790:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d8005d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3060fe0 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074477.874806:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074477.874814:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042d80 x1855007612291200/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074477.874836:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291200:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074477.874973:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.882570:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291200:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074477.882636:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdd:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074477.882669:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074477.882696:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074477.882758:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xdd:0x0](ffff8b100485e3c8) 00000008:00010000:1.0:1769074477.882791:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074473 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074477.882892:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074477.882909:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074477.882967:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.882977:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074477.882986:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074477.883007:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291328:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074477.883089:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074477.887278:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291328:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074477.887308:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887419:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887432:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887445:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887457:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887483:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887490:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074477.887501:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043100 x1855007612291328/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074477.887529:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887542:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074477.887548:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074477.887578:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:1.0:1769074477.887585:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80064 00000080:00010000:1.0:1769074477.887591:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074477.887595:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80064 from it ffffaa914211fb58 00010000:00010000:1.0:1769074477.887598:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074477.887608:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074477.887633:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074477.887826:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-18, flags=8 00000080:00200008:0.0:1769074477.888004:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-18:[0x200001b71:0x3c:0x0](ffff8b1018126c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.888011:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x3c:0x0](ffff8b1018126c08),name=f5.replay-single-18 00000002:00010000:0.0:1769074477.888032:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x3c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.888043:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.888055:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.888067:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.888071:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x3c:0x0](ffff8b1018126c08) 00000080:00010000:0.0:1769074477.888074:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x3c:0x0](ffff8b1018126c08) for lock 0x68a7113811d7f86f 00000080:00010000:0.0:1769074477.888078:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f86f from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.888081:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.888087:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.888107:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x3c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.888116:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x3c:0x0] 00000008:00010000:0.0:1769074477.888134:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.888137:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d430, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.888154:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.888163:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.888173:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.888176:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.888183:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.888196:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.888206:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x3c:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074477.888211:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.888213:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.888216:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d430 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.888220:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.888223:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d430 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.888233:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x3c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.888263:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.888271:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.888277:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.888285:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.888291:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.888295:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.888298:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80064 00000080:00010000:0.0:1769074477.888301:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80064 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.888304:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.888409:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.888420:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.888429:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-18, flags=0 00000080:00200000:0.0:1769074477.888434:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.888441:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.888447:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.888454:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.888459:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.888463:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.888466:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80064 00000080:00010000:0.0:1769074477.888469:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80064 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.888472:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.888477:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.888484:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.888489:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-18, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.888597:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.888871:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.888990:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x3c:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.889028:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.889040:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.889045:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.889048:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601dc00/0x68a7113811d7f86f lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x3c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e5cb expref: -99 pid: 30752 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.889064:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80064 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306100a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.889079:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.889089:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064ab800 x1855007612291584/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.889112:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064ab800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291584:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.889232:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.899152:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b100604ed80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291712:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.899284:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.904066:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b100604ed80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291712:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.904092:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904104:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904115:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904131:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904144:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904152:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904159:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.904169:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b100604ed80 x1855007612291712/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.904183:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904197:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.904204:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.904231:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.904238:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8006b 00000080:00010000:0.0:1769074477.904242:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.904246:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8006b from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.904248:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904257:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.904351:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.904363:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-180, flags=8 00000080:00200008:0.0:1769074477.904374:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-180:[0x200001b71:0xde:0x0](ffff8b1004851148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.904380:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xde:0x0](ffff8b1004851148),name=f5.replay-single-180 00000002:00010000:0.0:1769074477.904398:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xde:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.904408:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.904419:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.904426:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.904430:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xde:0x0](ffff8b1004851148) 00000080:00010000:0.0:1769074477.904433:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xde:0x0](ffff8b1004851148) for lock 0x68a7113811d7fcdd 00000080:00010000:0.0:1769074477.904437:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fcdd from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.904440:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.904446:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.904467:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xde:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074477.904478:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xde:0x0] 00000008:00010000:0.0:1769074477.904496:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.904499:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d430, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.904516:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.904531:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.904541:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.904544:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.904552:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.904561:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.904573:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xde:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.904577:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.904580:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.904583:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d430 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.904587:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.904589:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d430 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.904605:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xde:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.904751:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.904761:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.904768:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904776:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.904782:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.904786:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.904788:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8006b 00000080:00010000:0.0:1769074477.904792:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8006b from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.904795:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904801:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.904809:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.904817:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-180, flags=0 00000080:00200000:0.0:1769074477.904821:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.904827:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.904832:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904840:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.904845:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.904849:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.904851:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8006b 00000080:00010000:0.0:1769074477.904855:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8006b from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.904858:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904864:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.904872:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.904878:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-180, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.904902:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.904948:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.904958:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xde:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.904994:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.905001:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.905006:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.905009:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054400/0x68a7113811d7fcdd lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xde:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fbf1 expref: -99 pid: 30915 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.905027:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d8006b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061034 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.905045:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.905054:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10064a9180 x1855007612291968/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.905077:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291968:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.905214:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.911546:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10064a9180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612291968:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.911618:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xde:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.911632:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.911678:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xde:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.911747:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xde:0x0](ffff8b1004851148) 00000008:00010000:0.0:1769074477.911784:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074473 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.911872:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.911890:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.911954:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.912052:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.912062:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.912084:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292096:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.912214:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.918439:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292096:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.918465:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918475:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918484:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918492:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918501:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918507:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918514:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.918523:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c1180 x1855007612292096/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.918537:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918550:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.918557:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.918583:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.918589:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80072 00000080:00010000:0.0:1769074477.918593:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.918596:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80072 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.918598:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918604:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.918625:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.918634:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-181, flags=8 00000080:00200008:0.0:1769074477.918674:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-181:[0x200001b71:0xdf:0x0](ffff8b1004850908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.918679:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xdf:0x0](ffff8b1004850908),name=f5.replay-single-181 00000002:00010000:0.0:1769074477.918695:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xdf:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.918702:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.918721:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.918728:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.918732:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) 00000080:00010000:0.0:1769074477.918735:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xdf:0x0](ffff8b1004850908) for lock 0x68a7113811d7fce4 00000080:00010000:0.0:1769074477.918739:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fce4 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.918746:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.918754:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.918777:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.918788:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xdf:0x0] 00000008:00010000:0.0:1769074477.918808:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.918811:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.918831:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.918841:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.918857:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.918862:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.918869:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.918878:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.918892:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xdf:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.918896:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.918899:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.918903:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.918906:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.918909:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.918919:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.918953:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.918961:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.918968:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.918975:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.918981:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.918985:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.918987:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80072 00000080:00010000:0.0:1769074477.918993:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80072 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.918996:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.919002:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.919010:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.919016:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-181, flags=0 00000080:00200000:0.0:1769074477.919020:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.919026:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.919030:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.919036:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.919041:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.919044:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.919046:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80072 00000080:00010000:0.0:1769074477.919050:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80072 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.919052:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.919058:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.919064:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.919069:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-181, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.919090:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.919136:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.919147:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xdf:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.919182:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.919190:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.919195:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.919198:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e600/0x68a7113811d7fce4 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xdf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc14 expref: -99 pid: 30916 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.919215:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80072 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306105e expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.919232:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.919241:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c1180 x1855007612292352/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.919266:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292352:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.919352:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.926837:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292352:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.926908:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xdf:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.926925:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.926957:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.927014:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xdf:0x0](ffff8b1004850908) 00000008:00010000:0.0:1769074477.927049:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.927223:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.927241:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.927295:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.927304:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.927314:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.927345:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292480:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.927411:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.936262:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292480:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.936298:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936310:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936321:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936340:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936358:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936367:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936374:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.936384:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c0e00 x1855007612292480/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.936396:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936408:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.936681:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.936722:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.936729:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80079 00000080:00010000:0.0:1769074477.936735:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.936739:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80079 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.936742:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.936751:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.936775:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.936787:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-182, flags=8 00000080:00200008:0.0:1769074477.936799:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffff8b10048542c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.936808:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffff8b10048542c8),name=f5.replay-single-182 00000002:00010000:0.0:1769074477.936825:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.936835:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.936845:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.936851:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.936855:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) 00000080:00010000:0.0:1769074477.936857:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe0:0x0](ffff8b10048542c8) for lock 0x68a7113811d7fceb 00000080:00010000:0.0:1769074477.936861:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fceb from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.936864:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.936871:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.936891:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.936902:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe0:0x0] 00000008:00010000:0.0:1769074477.936922:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.936926:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.936944:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.936953:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.936968:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.936972:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.936979:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.936988:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.937001:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe0:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.937005:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.937008:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.937010:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.937014:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.937017:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.937026:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.937056:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.937064:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.937070:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937077:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.937084:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.937088:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.937091:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80079 00000080:00010000:0.0:1769074477.937096:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80079 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.937099:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937105:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.937113:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.937120:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-182, flags=0 00000080:00200000:0.0:1769074477.937124:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.937130:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.937136:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937142:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.937148:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.937151:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.937153:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80079 00000080:00010000:0.0:1769074477.937156:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80079 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.937159:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937165:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.937171:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.937176:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-182, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.937200:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937242:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.937253:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe0:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.937287:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.937294:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.937300:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.937303:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e200/0x68a7113811d7fceb lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc37 expref: -99 pid: 30917 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.937320:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80079 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061088 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.937350:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.937360:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c0e00 x1855007612292736/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.937381:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292736:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.937906:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.943979:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292736:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.944038:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.944053:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.944079:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.944125:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffff8b10048542c8) 00000008:00010000:0.0:1769074477.944157:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.944230:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.944249:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.944302:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.944310:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.944320:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.944339:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292864:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.944395:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.954517:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612292864:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.954545:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954558:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954569:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954581:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954596:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954603:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954610:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.954622:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2680 x1855007612292864/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.954634:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954682:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.954689:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.954830:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.954839:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80080 00000080:00010000:0.0:1769074477.954844:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.954847:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80080 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.954850:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.954857:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.954880:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.954892:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-183, flags=8 00000080:00200008:0.0:1769074477.954905:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-183:[0x200001b71:0xe1:0x0](ffff8b10048563c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.954911:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe1:0x0](ffff8b10048563c8),name=f5.replay-single-183 00000002:00010000:0.0:1769074477.954929:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.954939:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.954949:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.954955:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.954959:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe1:0x0](ffff8b10048563c8) 00000080:00010000:0.0:1769074477.954961:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe1:0x0](ffff8b10048563c8) for lock 0x68a7113811d7fcf2 00000080:00010000:0.0:1769074477.954965:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fcf2 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.954967:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.954973:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.954991:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.955001:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe1:0x0] 00000008:00010000:0.0:1769074477.955018:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.955021:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.955037:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1e00/0x68a7113811d7f48d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.955046:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1e00/0x68a7113811d7f48d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.955062:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.955065:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1e00/0x68a7113811d7f48d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bf8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.955071:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1e00/0x68a7113811d7f48d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.955080:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.955092:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe1:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.955096:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.955098:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.955101:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.955105:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.955108:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.955117:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.955150:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.955157:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.955164:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955172:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.955177:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.955181:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.955183:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80080 00000080:00010000:0.0:1769074477.955189:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80080 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.955192:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955198:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.955205:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.955213:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-183, flags=0 00000080:00200000:0.0:1769074477.955217:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.955221:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.955227:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955233:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.955239:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.955242:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.955244:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80080 00000080:00010000:0.0:1769074477.955247:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80080 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.955250:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955255:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.955262:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.955268:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-183, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.955289:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955344:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.955355:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe1:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.955388:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.955394:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.955398:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.955401:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e400/0x68a7113811d7fcf2 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc5a expref: -99 pid: 30918 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.955418:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80080 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30610b2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.955435:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.955443:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2680 x1855007612293120/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.955464:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293120:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.955538:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.968890:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293120:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.968960:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe1:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.968974:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.969002:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.969060:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe1:0x0](ffff8b10048563c8) 00000008:00010000:0.0:1769074477.969094:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1e00/0x68a7113811d7f48d lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bf8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.969183:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.969204:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.969264:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.969273:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.969284:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.969305:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293248:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.969381:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.977733:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293248:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074477.977764:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977776:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977788:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977800:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977817:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977824:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977830:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074477.977840:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2300 x1855007612293248/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074477.977852:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977865:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074477.977871:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074477.977901:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074477.977908:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80087 00000080:00010000:0.0:1769074477.977913:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.977917:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80087 from it ffffaa914211fb58 00010000:00010000:0.0:1769074477.977920:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.977928:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.977953:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.977965:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-184, flags=8 00000080:00200008:0.0:1769074477.977976:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-184:[0x200001b71:0xe2:0x0](ffff8b1004851988), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074477.977982:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe2:0x0](ffff8b1004851988),name=f5.replay-single-184 00000002:00010000:0.0:1769074477.978000:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.978010:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.978021:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.978027:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.978031:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe2:0x0](ffff8b1004851988) 00000080:00010000:0.0:1769074477.978033:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe2:0x0](ffff8b1004851988) for lock 0x68a7113811d7fcf9 00000080:00010000:0.0:1769074477.978037:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fcf9 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074477.978040:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.978047:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.978067:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074477.978078:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe2:0x0] 00000008:00010000:0.0:1769074477.978095:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074477.978099:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074477.978115:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.978125:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.978142:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074477.978146:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074477.978153:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074477.978162:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074477.978174:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe2:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074477.978179:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074477.978181:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.978184:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074477.978188:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074477.978190:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074477.978201:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe2:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.978233:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.978241:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.978247:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978255:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.978261:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.978265:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.978268:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80087 00000080:00010000:0.0:1769074477.978273:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80087 from it ffffaa914211fc10 00010000:00010000:0.0:1769074477.978277:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978283:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.978292:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074477.978298:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-184, flags=0 00000080:00200000:0.0:1769074477.978302:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.978308:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.978313:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978321:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.978326:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074477.978330:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074477.978332:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80087 00000080:00010000:0.0:1769074477.978335:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80087 from it ffffaa914211fd68 00010000:00010000:0.0:1769074477.978338:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978343:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074477.978349:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074477.978354:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-184, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074477.978375:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978429:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074477.978441:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe2:0x0]: losing layout lock 00010000:00010000:0.0:1769074477.978473:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.978479:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074477.978484:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074477.978487:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055800/0x68a7113811d7fcf9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fc7d expref: -99 pid: 30919 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074477.980189:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80087 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30610dc expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074477.980231:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074477.980242:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2300 x1855007612293504/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074477.980271:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293504:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074477.980371:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074477.996746:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293504:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074477.996820:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe2:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074477.996835:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074477.996861:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074477.996910:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe2:0x0](ffff8b1004851988) 00000008:00010000:0.0:1769074477.996944:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074477.997025:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074477.997044:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074477.997099:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.997108:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074477.997117:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074477.997135:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293632:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074477.997192:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.001393:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293632:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074478.001435:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001447:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001458:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001470:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001486:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001493:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001499:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074478.001520:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c3800 x1855007612293632/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074493 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074478.001533:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001546:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074478.001552:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074478.001579:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074478.001584:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8008e 00000080:00010000:0.0:1769074478.001589:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.001591:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8008e from it ffffaa914211fb58 00010000:00010000:0.0:1769074478.001594:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001602:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.001625:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.001679:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-185, flags=8 00000080:00200008:0.0:1769074478.001694:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-185:[0x200001b71:0xe3:0x0](ffff8b1004857448), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074478.001699:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe3:0x0](ffff8b1004857448),name=f5.replay-single-185 00000002:00010000:0.0:1769074478.001728:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe3:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.001738:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.001749:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.001756:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.001760:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe3:0x0](ffff8b1004857448) 00000080:00010000:0.0:1769074478.001763:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe3:0x0](ffff8b1004857448) for lock 0x68a7113811d7fd00 00000080:00010000:0.0:1769074478.001767:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd00 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074478.001769:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.001776:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.001795:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074478.001807:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe3:0x0] 00000008:00010000:0.0:1769074478.001827:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074478.001830:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074478.001848:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.001859:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.001876:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074478.001879:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.001886:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.001896:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074478.001908:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe3:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074478.001912:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074478.001915:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.001918:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.001922:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074478.001925:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074478.001935:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe3:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.001969:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.001977:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.001984:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.001992:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.001998:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.002002:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.002004:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8008e 00000080:00010000:0.0:1769074478.002010:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8008e from it ffffaa914211fc10 00010000:00010000:0.0:1769074478.002013:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.002020:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.002028:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.002035:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-185, flags=0 00000080:00200000:0.0:1769074478.002039:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.002045:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.002051:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.002057:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.002063:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.002067:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.002069:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8008e 00000080:00010000:0.0:1769074478.002072:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8008e from it ffffaa914211fd68 00010000:00010000:0.0:1769074478.002075:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.002081:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.002089:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074478.002094:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-185, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074478.002117:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.002158:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.002169:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe3:0x0]: losing layout lock 00010000:00010000:0.0:1769074478.002200:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.002208:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.002213:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.002216:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603e800/0x68a7113811d7fd00 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fca0 expref: -99 pid: 30920 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.002234:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8008e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061106 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.002251:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.002260:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c3800 x1855007612293888/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074478.002282:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293888:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074478.002359:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.006885:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612293888:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.006956:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe3:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.006972:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.007001:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.007055:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe3:0x0](ffff8b1004857448) 00000008:00010000:0.0:1769074478.007089:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1000/0x68a7113811d7f4b7 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305bffa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.007176:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.007195:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.007260:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.007270:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.007280:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.007302:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294016:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.007394:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.011425:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294016:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074478.011451:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011464:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011475:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011487:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011517:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011525:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011532:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074478.011543:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2a00 x1855007612294016/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074478.011555:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011567:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074478.011574:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074478.011602:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074478.011607:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80095 00000080:00010000:0.0:1769074478.011612:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.011616:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80095 from it ffffaa914211fb58 00010000:00010000:0.0:1769074478.011619:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.011627:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.011691:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.011715:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-186, flags=8 00000080:00200008:0.0:1769074478.011728:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-186:[0x200001b71:0xe4:0x0](ffff8b1004853a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074478.011734:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe4:0x0](ffff8b1004853a88),name=f5.replay-single-186 00000002:00010000:0.0:1769074478.011753:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe4:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.011765:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.011774:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.011781:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.011785:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe4:0x0](ffff8b1004853a88) 00000080:00010000:0.0:1769074478.011788:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe4:0x0](ffff8b1004853a88) for lock 0x68a7113811d7fd07 00000080:00010000:0.0:1769074478.011792:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd07 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074478.011795:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.011803:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.011826:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074478.011837:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe4:0x0] 00000008:00010000:0.0:1769074478.011856:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074478.011860:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074478.011877:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.011886:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.011903:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074478.011907:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.011915:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.011925:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074478.011938:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe4:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074478.011943:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074478.011946:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.011949:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.011954:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074478.011957:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074478.011967:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe4:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.012001:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.012010:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.012017:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012025:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.012031:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.012035:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.012038:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80095 00000080:00010000:0.0:1769074478.012043:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80095 from it ffffaa914211fc10 00010000:00010000:0.0:1769074478.012046:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012053:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.012061:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.012067:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-186, flags=0 00000080:00200000:0.0:1769074478.012071:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.012077:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.012083:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012089:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.012095:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.012098:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.012100:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80095 00000080:00010000:0.0:1769074478.012103:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80095 from it ffffaa914211fd68 00010000:00010000:0.0:1769074478.012106:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012113:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.012121:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074478.012125:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-186, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074478.012148:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012189:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.012198:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe4:0x0]: losing layout lock 00010000:00010000:0.0:1769074478.012229:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.012235:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.012240:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.012242:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006054200/0x68a7113811d7fd07 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fcc3 expref: -99 pid: 30921 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.012260:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80095 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061130 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.012278:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.012287:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2a00 x1855007612294272/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074478.012310:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294272:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074478.012384:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.018507:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294272:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.018575:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe4:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.018590:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.018617:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.018748:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe4:0x0](ffff8b1004853a88) 00000008:00010000:0.0:1769074478.018780:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.018861:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.018879:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.018935:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.018943:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.018953:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.018973:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294400:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.019029:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.024056:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294400:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074478.024082:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024094:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024106:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024117:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024133:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024140:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024146:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074478.024155:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c3100 x1855007612294400/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074478.024168:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024179:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074478.024185:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074478.024212:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074478.024218:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8009c 00000080:00010000:0.0:1769074478.024222:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.024225:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8009c from it ffffaa914211fb58 00010000:00010000:0.0:1769074478.024228:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024236:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.024259:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.024271:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-187, flags=8 00000080:00200008:0.0:1769074478.024284:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-187:[0x200001b71:0xe5:0x0](ffff8b1004862a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074478.024290:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe5:0x0](ffff8b1004862a08),name=f5.replay-single-187 00000002:00010000:0.0:1769074478.024319:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.024337:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.024347:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.024354:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.024359:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) 00000080:00010000:0.0:1769074478.024361:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe5:0x0](ffff8b1004862a08) for lock 0x68a7113811d7fd0e 00000080:00010000:0.0:1769074478.024365:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd0e from it ffffaa914211fcb8 00010000:00010000:0.0:1769074478.024367:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.024373:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.024394:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074478.024406:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe5:0x0] 00000008:00010000:0.0:1769074478.024424:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074478.024428:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d0b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074478.024443:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.024454:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.024469:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074478.024473:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.024481:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.024491:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074478.024501:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe5:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074478.024505:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074478.024509:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.024512:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d0b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.024516:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074478.024518:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d0b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074478.024527:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.024560:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.024568:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.024574:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024583:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.024589:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.024593:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.024595:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8009c 00000080:00010000:0.0:1769074478.024600:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8009c from it ffffaa914211fc10 00010000:00010000:0.0:1769074478.024603:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024609:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.024617:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.024623:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-187, flags=0 00000080:00200000:0.0:1769074478.024628:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.024633:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.024667:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024676:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.024681:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.024685:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.024687:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8009c 00000080:00010000:0.0:1769074478.024691:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8009c from it ffffaa914211fd68 00010000:00010000:0.0:1769074478.024694:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024700:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.024721:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074478.024726:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-187, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074478.024749:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024792:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.024803:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe5:0x0]: losing layout lock 00010000:00010000:0.0:1769074478.024835:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.024842:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.024847:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.024850:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100164a800/0x68a7113811d7fd0e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fce6 expref: -99 pid: 30922 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.024867:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8009c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306115a expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.024884:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.024893:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c3100 x1855007612294656/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074478.024917:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294656:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074478.024993:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.032843:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294656:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.032912:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.032927:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.032956:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.033007:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe5:0x0](ffff8b1004862a08) 00000008:00010000:0.0:1769074478.033040:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e0200/0x68a7113811d7f4e1 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c06a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.033123:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.033140:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.033195:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.033204:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.033212:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.033233:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294784:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.033302:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.036880:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612294784:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074478.036910:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036923:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036934:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036946:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036962:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036969:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.036977:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074478.036987:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2d80 x1855007612294784/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074478.037000:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037013:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074478.037020:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074478.037047:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 30208 00000080:00010000:0.0:1769074478.037053:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d800a3 00000080:00010000:0.0:1769074478.037058:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.037062:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d800a3 from it ffffaa914211fb58 00010000:00010000:0.0:1769074478.037065:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037074:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.037098:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.037110:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-188, flags=8 00000080:00200008:0.0:1769074478.037122:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-188:[0x200001b71:0xe6:0x0](ffff8b1004866c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074478.037127:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe6:0x0](ffff8b1004866c08),name=f5.replay-single-188 00000002:00010000:0.0:1769074478.037144:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xe6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.037154:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.037165:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.037173:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.037177:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe6:0x0](ffff8b1004866c08) 00000080:00010000:0.0:1769074478.037180:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe6:0x0](ffff8b1004866c08) for lock 0x68a7113811d7fd15 00000080:00010000:0.0:1769074478.037184:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd15 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074478.037186:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.037193:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.037213:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074478.037224:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe6:0x0] 00000008:00010000:0.0:1769074478.037243:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074478.037246:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d0b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074478.037263:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.037274:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.037290:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074478.037294:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.037301:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.037311:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074478.037323:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xe6:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074478.037341:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_release():245 00000020:00010000:0.0:1769074478.037344:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.037347:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d0b0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.037351:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1009438920 (ffff8b1007c69248/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074478.037353:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d0b0 (ffff8b1007c69248/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074478.037363:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe6:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.037396:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.037406:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.037412:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037420:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.037427:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.037430:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.037433:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d800a3 00000080:00010000:0.0:1769074478.037439:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d800a3 from it ffffaa914211fc10 00010000:00010000:0.0:1769074478.037442:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037449:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.037457:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.037465:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-188, flags=0 00000080:00200000:0.0:1769074478.037469:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.037475:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.037480:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037487:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.037492:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.037496:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.037498:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d800a3 00000080:00010000:0.0:1769074478.037501:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d800a3 from it ffffaa914211fd68 00010000:00010000:0.0:1769074478.037504:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037511:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.037518:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074478.037524:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-188, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074478.037547:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037586:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.037596:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xe6:0x0]: losing layout lock 00010000:00010000:0.0:1769074478.037629:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.037928:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.037937:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.037940:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006055200/0x68a7113811d7fd15 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305fd09 expref: -99 pid: 30923 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.037958:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d800a3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061184 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.037977:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.037986:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2d80 x1855007612295040/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074478.038863:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612295040:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074478.038951:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074478.047836:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612295040:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.047905:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xe6:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.047920:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.047950:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xe6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.048002:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xe6:0x0](ffff8b1004866c08) 00000008:00010000:0.0:1769074478.048041:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074413 1769074413 0 by oinfo size 8 blocks 1 [cma]time 1769074413 1769074413 1769074474 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074478.058693:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074478.058738:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074478.058842:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074478.058855:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074478.058868:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074478.058908:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612295168:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074478.059492:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.066143:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612295168:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074478.066172:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066186:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066198:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066210:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066222:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066230:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066253:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074478.066276:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043100 x1855007612295168/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074478.066290:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d800aa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30611ae expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.066312:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074478.066320:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:1.0:1769074478.313246:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.468413:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612301952:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074478.468480:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074478.468493:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074478.468524:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074478.468576:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xf5:0x0](ffff8b100485ba88) 00000008:00010000:1.0:1769074478.468606:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074414 1769074414 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 1769074475 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x227:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074478.468872:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074478.468891:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074478.468949:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074478.469020:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074478.469049:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074478.469073:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612302080:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074478.469195:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.486136:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612302080:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074478.486166:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486179:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486190:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486202:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486215:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486223:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486230:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074478.486242:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043800 x1855007612302080/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074478.486278:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.486293:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074478.486299:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074478.487140:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074478.487155:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80128 00000080:00010000:1.0:1769074478.487162:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074478.487166:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80128 from it ffffaa914211fb58 00010000:00010000:1.0:1769074478.487170:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.487182:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074478.487212:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074478.487223:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-204, flags=8 00000080:00200008:1.0:1769074478.487236:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-204:[0x200001b71:0xf6:0x0](ffff8b10048600c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074478.487241:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xf6:0x0](ffff8b10048600c8),name=f5.replay-single-204 00000002:00010000:1.0:1769074478.487262:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xf6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074478.487274:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074478.487286:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074478.487294:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074478.487300:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf6:0x0](ffff8b10048600c8) 00000080:00010000:1.0:1769074478.487312:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf6:0x0](ffff8b10048600c8) for lock 0x68a7113811d7fd85 00000080:00010000:1.0:1769074478.487317:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd85 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074478.487321:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074478.487661:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074478.487687:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074478.487708:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf6:0x0] 00000008:00010000:1.0:1769074478.487728:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074478.487732:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f29b0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074478.487750:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.487761:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074478.487773:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074478.487777:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.487785:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074478.487794:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074478.487807:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xf6:0x0] updating i_size 8 i_blocks 1 00000020:00010000:1.0:1769074478.487811:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074478.487814:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074478.487823:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f29b0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074478.487827:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074478.487830:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f29b0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074478.487842:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf6:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074478.487878:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074478.487887:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074478.487894:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.487903:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074478.487910:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074478.487915:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074478.487918:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80128 00000080:00010000:1.0:1769074478.487921:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80128 from it ffffaa914211fc10 00010000:00010000:1.0:1769074478.487924:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.487931:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074478.487940:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074478.487947:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-204, flags=0 00000080:00200000:1.0:1769074478.487952:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074478.487958:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074478.487963:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.487982:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074478.487988:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074478.487992:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074478.487995:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80128 00000080:00010000:1.0:1769074478.487998:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80128 from it ffffaa914211fd68 00010000:00010000:1.0:1769074478.488002:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.488009:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074478.488017:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074478.488022:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-204, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074478.488045:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.488085:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074478.488096:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xf6:0x0]: losing layout lock 00010000:00010000:1.0:1769074478.488132:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074478.488139:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074478.488145:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074478.488154:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060e0600/0x68a7113811d7fd85 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xf6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ff39 expref: -99 pid: 30939 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074478.488174:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80128 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30614a2 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074478.488191:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074478.488202:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043800 x1855007612302336/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074478.488222:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612302336:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074478.488310:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.507834:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612302336:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.509443:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf6:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.509462:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.509491:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.509551:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xf6:0x0](ffff8b10048600c8) 00000008:00010000:0.0:1769074478.509589:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074414 1769074414 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 1769074475 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.509795:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.509814:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.509878:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8012f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.509887:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8012f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.509896:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8012f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.509921:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612302464:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.510004:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.539291:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.539356:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608000 00000100:00100000:1.0:1769074478.539370:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608000 00000100:00100000:1.0:1769074478.539439:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 18 00000100:00100000:1.0:1769074478.539451:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:32370:x1855007624608000:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.539474:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.539588:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.539596:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:32370:x1855007624608000:12345-192.168.204.102@tcp:104: Request processed in 147us (322us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.539603:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 18 00000100:00100000:1.0:1769074478.539722:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.539826:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.539855:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.539915:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.539933:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608128 00000100:00100000:1.0:1769074478.539938:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608128 00000100:00100000:1.0:1769074478.539956:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608256 00000100:00100000:1.0:1769074478.539959:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608256 00000100:00100000:1.0:1769074478.539974:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 19 00000100:00100000:1.0:1769074478.539981:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624608128:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.539992:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540033:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.540039:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624608128:12345-192.168.204.102@tcp:104: Request processed in 58us (317us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.540046:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 19 00000100:00100000:1.0:1769074478.540060:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 20 00000100:00100000:1.0:1769074478.540068:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10032cf050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624608256:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.540077:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540109:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.540114:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10032cf050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624608256:12345-192.168.204.102@tcp:104: Request processed in 48us (288us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.540120:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 20 00010000:00010000:1.0:1769074478.540196:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.540205:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540213:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10087f1a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.540228:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540289:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540297:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.540327:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540334:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1a00/0x68a7113811d7e65c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30599b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540360:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.540364:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540370:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100565f800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.540373:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540391:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540398:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.540405:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540411:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f800/0x68a7113811d7ee3c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aeb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540432:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.540435:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540441:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e0200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.540445:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540468:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540474:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.540480:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540486:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0200/0x68a7113811d7e59f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.540501:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.540503:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.540505:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.540525:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612302976:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.540587:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612303232:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.540617:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.540623:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612303104:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.544034:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005687800/0x68a7113811d7fd93 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ff7f expref: -99 pid: 30941 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.544053:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80136 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30614f6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.544059:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.544063:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005687800/0x68a7113811d7fd93 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xf8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ff7f expref: -99 pid: 30941 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.544088:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80136 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30614f6 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.544109:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.544121:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c0700 x1855007612303616/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074478.544145:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612303616:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074478.544235:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.549967:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.550024:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608512 00000100:00100000:1.0:1769074478.550033:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608512 00000100:00100000:1.0:1769074478.550107:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 21 00000100:00100000:0.0:1769074478.550112:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612303104:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.550127:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6521:x1855007624608512:12345-192.168.204.102@tcp:104: 00000100:00100000:0.0:1769074478.550148:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612303232:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.550156:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654400/0x68a7113811d7ee12 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ae42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.550273:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.550282:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6521:x1855007624608512:12345-192.168.204.102@tcp:104: Request processed in 162us (333us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.550291:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 21 00000100:00100000:1.0:1769074478.556935:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.556991:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.557029:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.557031:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612302976:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.557067:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608640 00000100:00100000:1.0:1769074478.557079:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608640 00000100:00100000:0.0:1769074478.557107:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608768 00000100:00100000:0.0:1769074478.557117:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608768 00000100:00100000:1.0:1769074478.557138:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624608896 00000100:00100000:1.0:1769074478.557146:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624608896 00000100:00100000:0.0:1769074478.557176:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 22 00000100:00100000:0.0:1769074478.557191:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624608640:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.557208:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565ec00/0x68a7113811d7eeba lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b002 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.557308:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.557315:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624608640:12345-192.168.204.102@tcp:104: Request processed in 130us (396us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.557323:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 22 00000100:00100000:0.0:1769074478.557346:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 23 00000100:00100000:0.0:1769074478.557352:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8372:x1855007624608768:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.557362:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3be00/0x68a7113811d7ee27 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ae7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.557995:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.558005:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8372:x1855007624608768:12345-192.168.204.102@tcp:104: Request processed in 650us (1011us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.558015:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 23 00000100:00100000:0.0:1769074478.558047:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 24 00000100:00100000:0.0:1769074478.558069:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624608896:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.558195:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e600/0x68a7113811d7eea5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305afca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.558271:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.558281:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624608896:12345-192.168.204.102@tcp:104: Request processed in 216us (1253us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.558288:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 24 00000100:00100000:0.0:1769074478.567357:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.567548:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.567714:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624609536 00000100:00100000:1.0:1769074478.567733:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624609536 00000100:00100000:1.0:1769074478.567784:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624609664 00000100:00100000:1.0:1769074478.567787:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624609664 00000100:00100000:1.0:1769074478.567811:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 25 00000100:00100000:1.0:1769074478.567822:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6521:x1855007624609536:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.567846:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.567995:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.568002:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6521:x1855007624609536:12345-192.168.204.102@tcp:104: Request processed in 183us (657us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.568010:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 25 00000100:00100000:1.0:1769074478.568033:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 26 00000100:00100000:1.0:1769074478.568040:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33934:x1855007624609664:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.568049:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568081:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.568087:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33934:x1855007624609664:12345-192.168.204.102@tcp:104: Request processed in 47us (542us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.568093:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 26 00010000:00010000:1.0:1769074478.568115:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.568122:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568131:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005657200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.568137:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568194:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568202:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.568233:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568247:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657200/0x68a7113811d7edbe lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568286:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.568290:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568297:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100565fc00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.568301:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568593:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568617:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.568629:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568682:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565fc00/0x68a7113811d7ee66 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305af22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.568711:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.568713:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.568715:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.568737:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612304256:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.568805:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074478.568820:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612304384:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.572110:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.572149:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.572235:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.572299:0:30957:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477293 gen 1 00000100:00100000:0.0:1769074478.573866:0:30957:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037ddc00 x1855007612301952/t68719477294(68719477294) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074478.573890:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612303616:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.573974:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf8:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.573988:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.574016:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.574062:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xf8:0x0](ffff8b10048580c8) 00000100:00100000:1.0:1769074478.574683:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.574728:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624609920 00000100:00100000:1.0:1769074478.574738:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624609920 00000100:00100000:1.0:1769074478.574801:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 27 00000100:00100000:1.0:1769074478.574814:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624609920:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.574832:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656c00/0x68a7113811d7edd3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ad9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.574932:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.574943:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f6450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624609920:12345-192.168.204.102@tcp:104: Request processed in 131us (2843us total) trans 0 rc 0/0 00000008:00010000:0.0:1769074478.575304:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074414 1769074414 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 1769074475 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x228:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.575406:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 27 00000080:00200000:0.0:1769074478.575800:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.575820:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.575863:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.575872:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.575882:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.575898:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612304896:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.575927:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.583051:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.583103:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.583160:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624610560 00000100:00100000:0.0:1769074478.583179:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624610560 00000100:00100000:1.0:1769074478.583201:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.583247:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624610816 00000100:00100000:0.0:1769074478.583253:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624610816 00000100:00100000:0.0:1769074478.583280:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624610944 00000100:00100000:0.0:1769074478.583284:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624610944 00000100:00100000:0.0:1769074478.583316:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 28 00000100:00100000:0.0:1769074478.583337:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624610560:12345-192.168.204.102@tcp:104: 00000100:00100000:1.0:1769074478.583355:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612304384:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.583357:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f400/0x68a7113811d7ee90 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305af92 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.583378:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612304256:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.583471:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.583479:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624610560:12345-192.168.204.102@tcp:104: Request processed in 154us (446us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.583487:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 28 00000100:00100000:0.0:1769074478.583506:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 29 00000100:00100000:0.0:1769074478.583512:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33936:x1855007624610816:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.583521:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100565e400/0x68a7113811d7edfd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ae0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.583549:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.583555:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33936:x1855007624610816:12345-192.168.204.102@tcp:104: Request processed in 43us (453us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.583561:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 29 00000100:00100000:0.0:1769074478.583571:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 30 00000100:00100000:0.0:1769074478.583577:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33934:x1855007624610944:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.583585:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f0c00/0x68a7113811d7e671 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30599ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.583611:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074478.583616:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33934:x1855007624610944:12345-192.168.204.102@tcp:104: Request processed in 39us (417us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.583622:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 30 00010000:00010000:1.0:1769074478.584600:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.589400:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.589446:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.589480:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624611200 00000100:00100000:1.0:1769074478.589493:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624611200 00000100:00100000:1.0:1769074478.589542:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624611328 00000100:00100000:1.0:1769074478.589544:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624611328 00000100:00100000:1.0:1769074478.589557:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 31 00000100:00100000:1.0:1769074478.589567:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33937:x1855007624611200:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.589605:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ae00/0x68a7113811d7eecf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b03a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.589751:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.589758:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33937:x1855007624611200:12345-192.168.204.102@tcp:104: Request processed in 193us (374us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.589766:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 31 00000100:00100000:1.0:1769074478.589788:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 32 00000100:00100000:1.0:1769074478.589794:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624611328:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.589803:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727400/0x68a7113811d7eef9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b0aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.589842:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.589848:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624611328:12345-192.168.204.102@tcp:104: Request processed in 54us (402us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.589854:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 32 00000100:00100000:1.0:1769074478.597068:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.597196:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624611712 00000100:00100000:0.0:1769074478.597212:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624611712 00000100:00100000:0.0:1769074478.597295:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 33 00000100:00100000:0.0:1769074478.597307:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624611712:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.597340:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100565f200/0x68a7113811d7eee4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b072 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.597439:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.597447:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624611712:12345-192.168.204.102@tcp:104: Request processed in 142us (397us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.597454:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 33 00000100:00100000:1.0:1769074478.599462:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.599547:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.599762:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.599803:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.599825:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624611840 00000100:00100000:1.0:1769074478.599833:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624611840 00000100:00100000:1.0:1769074478.599884:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 34 00000100:00100000:1.0:1769074478.599904:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33935:x1855007624611840:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.599922:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b000/0x68a7113811d7ef0e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b0e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600024:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.600034:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f5c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624611840:12345-192.168.204.102@tcp:104: Request processed in 137us (583us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.600044:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 34 00010000:00010000:1.0:1769074478.600111:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.600121:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654600/0x68a7113811d7ef38 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b152 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600164:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005654600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.600174:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654600/0x68a7113811d7ef38 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b152 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600277:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654600/0x68a7113811d7ef38 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b152 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600289:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.600317:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654600/0x68a7113811d7ef38 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b152 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600329:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654600/0x68a7113811d7ef38 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b152 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.600363:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.600367:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.600368:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.600379:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.600398:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612305920:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.604472:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612304896:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074478.604506:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604521:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604534:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604545:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604559:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604568:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604574:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074478.604591:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2d80 x1855007612304896/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074494 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074478.604607:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604623:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074478.604631:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074478.604717:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074478.604727:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8013d 00000080:00010000:0.0:1769074478.604735:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.604739:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8013d from it ffffaa914211fb58 00010000:00010000:0.0:1769074478.604744:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.604755:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.604796:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.604812:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-207, flags=8 00000080:00200008:0.0:1769074478.604827:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-207:[0x200001b71:0xf9:0x0](ffff8b1004863248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074478.604833:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xf9:0x0](ffff8b1004863248),name=f5.replay-single-207 00000002:00010000:0.0:1769074478.604857:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0xf9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.604869:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.604881:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.604888:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.604892:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf9:0x0](ffff8b1004863248) 00000080:00010000:0.0:1769074478.604895:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf9:0x0](ffff8b1004863248) for lock 0x68a7113811d7fd9a 00000080:00010000:0.0:1769074478.604899:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fd9a from it ffffaa914211fcb8 00010000:00010000:0.0:1769074478.604902:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.604909:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.604930:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074478.604944:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf9:0x0] 00000008:00010000:0.0:1769074478.604968:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074478.604971:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d830, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074478.604992:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.605002:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.605015:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074478.605019:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.605027:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074478.605044:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074478.605058:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0xf9:0x0] updating i_size 8 i_blocks 1 00000020:00010000:0.0:1769074478.605063:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074478.605066:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.605069:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d830 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074478.605073:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074478.605076:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d830 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074478.605089:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf9:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.605127:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.605135:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.605141:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605149:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.605155:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.605159:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.605161:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8013d 00000080:00010000:0.0:1769074478.605165:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8013d from it ffffaa914211fc10 00010000:00010000:0.0:1769074478.605168:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605175:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.605183:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074478.605193:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-207, flags=0 00000080:00200000:0.0:1769074478.605197:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.605202:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.605207:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605215:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.605221:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074478.605224:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074478.605227:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8013d 00000080:00010000:0.0:1769074478.605230:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8013d from it ffffaa914211fd68 00010000:00010000:0.0:1769074478.605233:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605240:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074478.605247:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074478.605253:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-207, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074478.605279:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605331:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074478.605343:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0xf9:0x0]: losing layout lock 00010000:00010000:0.0:1769074478.605387:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.605399:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074478.605405:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074478.605408:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100603ee00/0x68a7113811d7fd9a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ffa2 expref: -99 pid: 30942 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074478.605431:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8013d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30615b3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074478.605452:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074478.605462:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2d80 x1855007612306304/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074478.605490:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612306304:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074478.605602:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074478.606491:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.606528:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.606564:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624612480 00000100:00100000:1.0:1769074478.606576:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624612480 00000100:00100000:1.0:1769074478.606623:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624612736 00000100:00100000:1.0:1769074478.606625:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624612736 00000100:00100000:1.0:1769074478.606679:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 35 00000100:00100000:1.0:1769074478.606690:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624612480:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.606716:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005654e00/0x68a7113811d7ef23 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b11a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.606790:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.606797:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624612480:12345-192.168.204.102@tcp:104: Request processed in 109us (313us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.606806:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 35 00000100:00100000:1.0:1769074478.606834:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 36 00000100:00100000:1.0:1769074478.606848:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624612736:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.606863:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.606904:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.606911:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624612736:12345-192.168.204.102@tcp:104: Request processed in 67us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.606918:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 36 00010000:00010000:1.0:1769074478.606939:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.606950:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.606993:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e0c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.606999:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.607062:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.607071:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.607094:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.607103:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0c00/0x68a7113811d7e686 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.607125:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.607128:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.607131:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.607150:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612306560:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.611858:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.614212:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624612992 00000100:00100000:0.0:1769074478.614233:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624612992 00000100:00100000:1.0:1769074478.614475:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 37 00000100:00100000:1.0:1769074478.614491:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624612992:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.614514:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc800/0x68a7113811d7ef62 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b1c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.614631:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.614665:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624612992:12345-192.168.204.102@tcp:104: Request processed in 152us (2788us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.614703:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 37 00000100:00100000:0.0:1769074478.618827:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.618871:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.618902:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.619020:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624613120 00000100:00100000:1.0:1769074478.619034:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624613120 00000100:00100000:1.0:1769074478.619083:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624613248 00000100:00100000:1.0:1769074478.619086:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624613248 00000100:00100000:1.0:1769074478.619096:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624613376 00000100:00100000:1.0:1769074478.619098:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624613376 00000100:00100000:1.0:1769074478.619114:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 38 00000100:00100000:1.0:1769074478.619123:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624613120:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.619145:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726c00/0x68a7113811d7ef77 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b1fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619238:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.619245:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624613120:12345-192.168.204.102@tcp:104: Request processed in 123us (435us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.619263:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 38 00000100:00100000:1.0:1769074478.619289:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 39 00000100:00100000:1.0:1769074478.619301:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33934:x1855007624613248:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.619316:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619373:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.619382:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33934:x1855007624613248:12345-192.168.204.102@tcp:104: Request processed in 84us (509us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.619389:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 39 00000100:00100000:1.0:1769074478.619409:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 40 00000100:00100000:1.0:1769074478.619419:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33936:x1855007624613376:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.619452:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc600/0x68a7113811d7efb6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b2a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619506:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.619516:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33936:x1855007624613376:12345-192.168.204.102@tcp:104: Request processed in 98us (611us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.619537:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 40 00010000:00010000:1.0:1769074478.619560:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.619568:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619577:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057cde00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.619584:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619672:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619682:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.619718:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619752:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cde00/0x68a7113811d7ef8c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b232 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.619786:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.619790:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.619792:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.619825:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612306944:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.638830:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612305920:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.639134:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.639285:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.639313:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624613760 00000100:00100000:1.0:1769074478.639334:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624613760 00000100:00100000:0.0:1769074478.639373:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.639396:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614144 00000100:00100000:1.0:1769074478.639402:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614144 00000100:00100000:0.0:1769074478.639410:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.639422:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614272 00000100:00100000:1.0:1769074478.639427:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614272 00000100:00100000:0.0:1769074478.639454:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.639463:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 41 00000100:00100000:1.0:1769074478.639478:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:14023:x1855007624613760:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.639500:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.639604:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.639612:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:14023:x1855007624613760:12345-192.168.204.102@tcp:104: Request processed in 137us (488us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.639620:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 41 00000100:00100000:1.0:1769074478.639714:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614528 00000100:00100000:1.0:1769074478.639719:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614528 00000100:00100000:1.0:1769074478.639743:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 42 00000100:00100000:1.0:1769074478.639749:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6522:x1855007624614144:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.639758:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.639801:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.639808:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6522:x1855007624614144:12345-192.168.204.102@tcp:104: Request processed in 59us (446us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.639837:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 42 00000100:00100000:1.0:1769074478.639854:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 43 00000100:00100000:1.0:1769074478.639860:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8372:x1855007624614272:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.639870:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.639918:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.639924:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8372:x1855007624614272:12345-192.168.204.102@tcp:104: Request processed in 64us (516us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.639931:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 43 00000100:00100000:1.0:1769074478.639944:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 44 00000100:00100000:1.0:1769074478.639950:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624614528:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.639968:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640001:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074478.640006:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624614528:12345-192.168.204.102@tcp:104: Request processed in 57us (555us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.640012:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 44 00000100:00100000:1.0:1769074478.640054:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612306560:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.640078:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1769074478.640086:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640094:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005727a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.640109:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640163:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640170:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.640195:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640202:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727a00/0x68a7113811d7efa1 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b26a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640229:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.640233:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640239:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057c4200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.640242:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640260:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 2/0,0 mode: --/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640267:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.640272:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 1/0,0 mode: --/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640279:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4200/0x68a7113811d7efcb lrc: 0/0,0 mode: --/PW res: [0x280000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b2da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640303:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.640307:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640314:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057cc000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.640318:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640337:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640344:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.640349:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640356:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cc000/0x68a7113811d7eff5 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b34a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640371:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.640374:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640381:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057c4800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.640384:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640401:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640412:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.640418:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640425:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057c4800/0x68a7113811d7f01f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b3ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.640439:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.640441:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.640443:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.640463:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307328:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.640520:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307584:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.640542:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074478.640548:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307200:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.640565:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307456:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.641761:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.644595:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.644737:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.644769:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.645155:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614656 00000100:00100000:1.0:1769074478.645167:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614656 00000100:00100000:1.0:1769074478.645218:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614784 00000100:00100000:1.0:1769074478.645221:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614784 00000100:00100000:1.0:1769074478.645232:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624614912 00000100:00100000:1.0:1769074478.645234:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624614912 00000100:00100000:1.0:1769074478.645278:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 45 00000100:00100000:1.0:1769074478.645289:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624614656:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.645309:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.645748:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612306304:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074478.645813:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xf9:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074478.645827:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.645855:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xf9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.645910:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xf9:0x0](ffff8b1004863248) 00000008:00010000:0.0:1769074478.645943:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074414 1769074414 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 1769074475 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.646032:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.646051:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.646106:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80144 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.646116:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80144 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.646126:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80144 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.646147:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612307712:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.646285:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00010000:00010000:1.0:1769074478.646832:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.646845:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624614656:12345-192.168.204.102@tcp:104: Request processed in 1555us (2253us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.646855:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 45 00000100:00100000:1.0:1769074478.646891:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 46 00000100:00100000:1.0:1769074478.646902:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33939:x1855007624614784:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.646913:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.646944:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.646950:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33939:x1855007624614784:12345-192.168.204.102@tcp:104: Request processed in 52us (2212us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.646955:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 46 00000100:00100000:1.0:1769074478.646969:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 47 00000100:00100000:1.0:1769074478.646976:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624614912:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.646985:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647012:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.647017:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624614912:12345-192.168.204.102@tcp:104: Request processed in 43us (2248us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.647023:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 47 00010000:00010000:1.0:1769074478.647061:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.647068:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647078:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ac800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.647085:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647132:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647138:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.647161:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647168:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d7e69b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059a5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647189:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.647193:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647199:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057cce00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.647202:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647220:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647237:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.647244:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647264:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cce00/0x68a7113811d7f00a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b382 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647282:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.647286:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647292:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057c4c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.647296:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.647314:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.648821:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.652042:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.652073:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.652084:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4c00/0x68a7113811d7efe0 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b312 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.652131:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.652134:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.652161:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.652208:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612307840:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.652353:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308096:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.652411:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.652421:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612307968:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.653305:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.653412:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.653450:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.653485:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074478.653501:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624615680 00000100:00100000:0.0:1769074478.653510:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624615680 00000100:00100000:0.0:1769074478.653562:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 48 00000100:00100000:0.0:1769074478.653576:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f4450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33940:x1855007624615680:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.653593:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5800/0x68a7113811d7f034 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b3f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.653683:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612306944:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.653708:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.653719:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f4450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33940:x1855007624615680:12345-192.168.204.102@tcp:104: Request processed in 146us (423us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.653727:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 48 00010000:00010000:0.0:1769074478.654400:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.654418:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727200/0x68a7113811d7f073 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b49a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.654441:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005727200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.654452:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727200/0x68a7113811d7f073 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b49a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.654510:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727200/0x68a7113811d7f073 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b49a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.654519:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.654537:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727200/0x68a7113811d7f073 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b49a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.654543:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727200/0x68a7113811d7f073 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b49a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.654564:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.654566:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.654568:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.656040:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.656070:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308480:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.656159:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.659658:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.659756:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.659785:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624616320 00000100:00100000:0.0:1769074478.659794:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624616320 00000100:00100000:0.0:1769074478.659831:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624616448 00000100:00100000:0.0:1769074478.659834:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624616448 00000100:00100000:0.0:1769074478.659848:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 49 00000100:00100000:0.0:1769074478.659868:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33941:x1855007624616320:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.659883:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.659954:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.659961:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33941:x1855007624616320:12345-192.168.204.102@tcp:104: Request processed in 106us (328us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.659969:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 49 00000100:00100000:0.0:1769074478.659988:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 50 00000100:00100000:0.0:1769074478.659994:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624616448:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.660003:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.660029:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.660034:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624616448:12345-192.168.204.102@tcp:104: Request processed in 40us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.660040:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 50 00000100:00100000:0.0:1769074478.660072:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.660128:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.660200:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624616576 00000100:00100000:0.0:1769074478.660204:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624616576 00000100:00100000:0.0:1769074478.660217:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624616704 00000100:00100000:0.0:1769074478.660219:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624616704 00000100:00100000:0.0:1769074478.660231:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 51 00000100:00100000:0.0:1769074478.660236:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624616576:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.660244:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.660275:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074478.660280:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624616576:12345-192.168.204.102@tcp:104: Request processed in 44us (209us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.660287:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 51 00000100:00100000:0.0:1769074478.660298:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 52 00000100:00100000:0.0:1769074478.660305:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624616704:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.660312:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.660336:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074478.660342:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624616704:12345-192.168.204.102@tcp:104: Request processed in 37us (214us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.660349:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 52 00000100:00100000:1.0:1769074478.660459:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307200:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.660478:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307456:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.660490:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307328:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.660499:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612307584:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.660542:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1769074478.660551:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660561:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057cc400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.660569:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660631:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660675:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.660710:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660718:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057cc400/0x68a7113811d7f05e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b462 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660748:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.660753:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660760:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057c4600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.660764:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660784:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660790:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.660795:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660802:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c4600/0x68a7113811d7f088 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b4d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660817:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.660821:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660836:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10056d9800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.660840:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660859:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660866:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.660871:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660879:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d9800/0x68a7113811d7f0b2 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b542 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660895:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.660899:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660905:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10056d9a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.660908:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660929:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660934:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.660939:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660952:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9a00/0x68a7113811d7f09d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b50a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.660984:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.660986:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.660988:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.661008:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308864:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.661084:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612309120:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.661110:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074478.661116:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308992:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.661136:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612309248:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.663315:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.669856:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.669893:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.669953:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.669970:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.670001:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.670023:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.670056:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074478.670073:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624617472 00000100:00100000:0.0:1769074478.670082:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624617472 00000100:00100000:0.0:1769074478.670119:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624616832 00000100:00100000:0.0:1769074478.670122:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624616832 00000100:00100000:0.0:1769074478.670132:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624617600 00000100:00100000:0.0:1769074478.670134:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624617600 00000100:00100000:0.0:1769074478.670146:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 53 00000100:00100000:0.0:1769074478.670154:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33942:x1855007624617472:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.670170:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.670232:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.670239:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33942:x1855007624617472:12345-192.168.204.102@tcp:104: Request processed in 86us (395us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.670247:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 53 00000100:00100000:0.0:1769074478.670264:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 54 00000100:00100000:0.0:1769074478.670270:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624616832:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.670279:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.670306:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.670312:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624616832:12345-192.168.204.102@tcp:104: Request processed in 42us (419us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.670333:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 54 00000100:00100000:0.0:1769074478.670349:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 55 00000100:00100000:0.0:1769074478.670357:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b100339b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624617600:12345-192.168.204.102@tcp:104: 00000100:00100000:1.0:1769074478.670464:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612307968:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.670491:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612307840:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.670513:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308096:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.670573:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.670585:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670627:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10056d9e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.670666:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670798:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670808:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.670841:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670858:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9e00/0x68a7113811d7f0c7 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b57a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670901:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074478.670907:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670916:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10056d8e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.670927:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670978:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 2/0,0 mode: --/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.670987:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.670996:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 1/0,0 mode: --/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671016:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8e00/0x68a7113811d7f0dc lrc: 0/0,0 mode: --/PW res: [0x240000400:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b5b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671063:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.671069:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cd800/0x68a7113811d7f145 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b6ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671079:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057cd800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.671084:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cd800/0x68a7113811d7f145 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b6ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671113:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cd800/0x68a7113811d7f145 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b6ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671122:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.671130:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cd800/0x68a7113811d7f145 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b6ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671146:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10057cd800/0x68a7113811d7f145 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b6ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671172:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.671178:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671185:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603f800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.671190:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671215:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671236:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.671243:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671262:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f800/0x68a7113811d7f16f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b73a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.671283:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.671286:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.671290:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.671310:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309504:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.671376:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309888:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.671404:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074478.671411:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309376:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.671429:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309760:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.671658:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10056d9600/0x68a7113811d7f0f1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b5ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.671718:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.671726:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b100339b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624617600:12345-192.168.204.102@tcp:104: Request processed in 1369us (1772us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.671733:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 55 00000100:00100000:0.0:1769074478.675185:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.675281:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.675310:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308480:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.675323:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624618624 00000100:00100000:0.0:1769074478.675348:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624618624 00000100:00100000:1.0:1769074478.675366:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624618880 00000100:00100000:1.0:1769074478.675374:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624618880 00000100:00100000:0.0:1769074478.675407:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 56 00000100:00100000:0.0:1769074478.676096:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33934:x1855007624618624:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.676112:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654200/0x68a7113811d7f106 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b622 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.676169:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.676176:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33934:x1855007624618624:12345-192.168.204.102@tcp:104: Request processed in 760us (1003us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.685892:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.686088:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.686099:0:13987:0:(service.c:3493:ptlrpc_start_thread()) starting thread 'ldlm_cb00_002' 00000100:00100000:0.0:1769074478.687547:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 56 00000100:00100000:0.0:1769074478.687592:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624619008 00000100:00100000:0.0:1769074478.687601:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624619008 00000100:00100000:0.0:1769074478.687661:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 57 00000100:00100000:0.0:1769074478.687671:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624618880:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.687689:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8000/0x68a7113811d7f130 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b692 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.687791:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.687798:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624618880:12345-192.168.204.102@tcp:104: Request processed in 129us (12517us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.687806:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 57 00000100:00100000:0.0:1769074478.687822:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 58 00000100:00100000:0.0:1769074478.687829:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624619008:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.687837:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cacc00/0x68a7113811d7e6b0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059a92 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.691363:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.691397:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.691410:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624619392 00000100:00100000:1.0:1769074478.691441:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624619392 00010000:00010000:0.0:1769074478.691536:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.691559:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624619008:12345-192.168.204.102@tcp:104: Request processed in 3722us (5675us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.691572:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 58 00000100:00100000:1.0:1769074478.691581:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308864:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.691626:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 59 00000100:00100000:0.0:1769074478.691662:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33943:x1855007624619392:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.691680:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603ea00/0x68a7113811d7f199 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b7aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.691705:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612309120:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695227:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612308992:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695334:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612309248:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695399:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309504:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695439:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309888:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695476:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309376:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695517:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612309760:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.695551:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.695577:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.695588:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624619520 00000100:00100000:1.0:1769074478.695595:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624619520 00000100:00100000:1.0:1769074478.695682:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.695706:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.695713:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624619648 00000100:00100000:1.0:1769074478.695716:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624619648 00000100:00100000:1.0:1769074478.695806:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.695828:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.695838:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.695842:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624620160 00000100:00100000:1.0:1769074478.695845:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624620160 00000100:00100000:1.0:1769074478.695884:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.695896:0:13987:0:(service.c:3434:ptlrpc_start_thread()) ldlm_cbd[0] started 2 min 2 max 34 00000100:00100000:1.0:1769074478.695920:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624620416 00000100:00100000:1.0:1769074478.695923:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624620416 00010000:00010000:0.0:1769074478.698600:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00010000:00010000:1.0:1769074478.698739:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.699066:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33943:x1855007624619392:12345-192.168.204.102@tcp:104: Request processed in 6978us (7265us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.699082:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 59 00000100:00100000:0.0:1769074478.699125:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 60 00000100:00100000:0.0:1769074478.699141:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33935:x1855007624619520:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.699160:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699264:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.699271:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33935:x1855007624619520:12345-192.168.204.102@tcp:104: Request processed in 139us (3721us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.699277:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 60 00000100:00100000:0.0:1769074478.699292:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 61 00000100:00100000:0.0:1769074478.699298:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006043b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:14023:x1855007624619648:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.699308:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699456:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.699462:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006043b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:14023:x1855007624619648:12345-192.168.204.102@tcp:104: Request processed in 164us (3781us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.699469:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 61 00000100:00100000:0.0:1769074478.699481:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 62 00000100:00100000:0.0:1769074478.699488:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b100bbf3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624620160:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.699496:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699523:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074478.699528:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b100bbf3050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624620160:12345-192.168.204.102@tcp:104: Request processed in 41us (3723us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.699534:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 62 00010000:00010000:0.0:1769074478.699569:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074478.699575:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699584:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005726a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.699590:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699634:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699667:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.699701:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699710:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005726a00/0x68a7113811d7f1c3 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b81a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.699718:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1769074478.699735:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074478.699741:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699748:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603fa00 already unused, calling callback (ffffffffc109c1b0) 00000100:00100000:1.0:1769074478.699756:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612310656:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.699757:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699791:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699797:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1769074478.699868:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612310784:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.699975:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.699986:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fa00/0x68a7113811d7f15a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b702 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700008:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.700012:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700018:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603fc00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.700023:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700055:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700062:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.700085:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700093:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603fc00/0x68a7113811d7f184 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b772 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700106:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.700108:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.700111:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.700133:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 63 00000100:00100000:1.0:1769074478.700184:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612310912:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.700795:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6519:x1855007624620416:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.700810:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700884:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.700892:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6519:x1855007624620416:12345-192.168.204.102@tcp:104: Request processed in 751us (5007us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.700899:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 63 00010000:00010000:0.0:1769074478.700922:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.700927:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700934:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005cade00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.700939:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700976:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.700994:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.701010:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.701018:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cade00/0x68a7113811d7e6c5 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059aca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.701035:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.701037:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.701039:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.701469:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311040:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.711757:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.711801:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.711822:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.711855:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624620928 00000100:00100000:1.0:1769074478.711867:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624620928 00000100:00100000:1.0:1769074478.711918:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621056 00000100:00100000:1.0:1769074478.711921:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621056 00000100:00100000:1.0:1769074478.711934:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621312 00000100:00100000:1.0:1769074478.711937:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621312 00000100:00100000:1.0:1769074478.711955:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 64 00000100:00100000:1.0:1769074478.711981:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33938:x1855007624620928:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.712004:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.712095:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.712103:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33938:x1855007624620928:12345-192.168.204.102@tcp:104: Request processed in 125us (355us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.712111:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 64 00000100:00100000:1.0:1769074478.712132:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 65 00000100:00100000:1.0:1769074478.712157:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33939:x1855007624621056:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.712167:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.712205:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.712211:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33939:x1855007624621056:12345-192.168.204.102@tcp:104: Request processed in 55us (409us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.712217:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 65 00000100:00100000:1.0:1769074478.712230:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 66 00000100:00100000:1.0:1769074478.712239:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33937:x1855007624621312:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.712249:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.712279:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.712284:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33937:x1855007624621312:12345-192.168.204.102@tcp:104: Request processed in 48us (462us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.712290:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 66 00000100:00100000:1.0:1769074478.712336:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.712358:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621440 00000100:00100000:1.0:1769074478.712364:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621440 00000100:00100000:1.0:1769074478.712385:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 67 00000100:00100000:1.0:1769074478.712390:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624621440:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.712398:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.712429:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074478.712436:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624621440:12345-192.168.204.102@tcp:104: Request processed in 45us (99us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.712453:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 67 00000100:00100000:1.0:1769074478.712567:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.712588:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621696 00000100:00100000:1.0:1769074478.712592:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621696 00000100:00100000:1.0:1769074478.712614:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 68 00000100:00100000:1.0:1769074478.712620:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624621696:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.712629:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712632:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074478.712668:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712684:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603f200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.712703:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.712706:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074478.712720:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624621696:12345-192.168.204.102@tcp:104: Request processed in 97us (152us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.712728:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 68 00010000:00010000:0.0:1769074478.712783:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 2/0,0 mode: --/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.712796:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00010000:00010000:0.0:1769074478.712806:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1769074478.712834:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00010000:00010000:0.0:1769074478.712854:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 1/0,0 mode: --/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712867:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100603f200/0x68a7113811d7f1ed lrc: 0/0,0 mode: --/PW res: [0x280000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b88a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.712876:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612311296:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.712901:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074478.712908:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712916:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603f600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.712919:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712947:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712954:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1769074478.712973:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621824 00000100:00100000:1.0:1769074478.712979:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621824 00010000:00010000:0.0:1769074478.712984:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.712993:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603f600/0x68a7113811d7f1ae lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b7e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.712995:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311424:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.713028:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074478.713033:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713040:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006055400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.713045:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713073:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713079:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1769074478.713088:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 69 00000100:00100000:1.0:1769074478.713098:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624621824:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.713103:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.713109:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713113:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055400/0x68a7113811d7f241 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b96a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713148:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00000100:00100000:1.0:1769074478.713148:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612311552:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.713154:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713172:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100603e000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.713177:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713205:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713211:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.713227:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.713231:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311680:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.713248:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603e000/0x68a7113811d7f1d8 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b852 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713267:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.713272:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713280:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006054e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.713283:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713311:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713318:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.713324:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.713333:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624621824:12345-192.168.204.102@tcp:104: Request processed in 237us (547us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.713341:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 69 00010000:00010000:0.0:1769074478.713350:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713361:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054e00/0x68a7113811d7f26b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713387:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1769074478.713390:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:0.0:1769074478.713392:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.713402:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311808:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.713404:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006055c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.713417:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713447:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713453:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.713480:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713490:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006055c00/0x68a7113811d7f217 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.713499:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311936:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.713513:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.713517:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.713520:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.713680:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.713725:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624621952 00000100:00100000:0.0:1769074478.713734:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624621952 00000100:00100000:0.0:1769074478.713785:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 70 00000100:00100000:0.0:1769074478.713796:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6521:x1855007624621952:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.713809:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713878:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.713889:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6521:x1855007624621952:12345-192.168.204.102@tcp:104: Request processed in 94us (211us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.713897:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 70 00010000:00010000:0.0:1769074478.713918:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.713922:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713932:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10056d8200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.713937:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713963:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713971:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.713982:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.713990:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10056d8200/0x68a7113811d7f202 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b8c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.714006:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.714009:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.714012:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.714074:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074478.714088:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312064:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.714993:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.715015:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.715064:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624622208 00000100:00100000:0.0:1769074478.715070:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624622208 00000100:00100000:0.0:1769074478.715099:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624622592 00000100:00100000:0.0:1769074478.715102:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624622592 00000100:00100000:0.0:1769074478.715116:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 71 00000100:00100000:0.0:1769074478.715123:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33935:x1855007624622208:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.715134:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715183:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.715190:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33935:x1855007624622208:12345-192.168.204.102@tcp:104: Request processed in 67us (202us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.715197:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 71 00000100:00100000:0.0:1769074478.715210:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 72 00000100:00100000:0.0:1769074478.715216:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005661c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:14023:x1855007624622592:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.715225:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715263:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.715269:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005661c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:14023:x1855007624622592:12345-192.168.204.102@tcp:104: Request processed in 52us (252us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.715275:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 72 00010000:00010000:0.0:1769074478.715288:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074478.715293:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715300:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006054600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.715305:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715343:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715351:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.715364:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715372:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054600/0x68a7113811d7f22c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b932 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715390:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.715393:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715399:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006054800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.715403:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715421:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715428:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.715434:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715440:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006054800/0x68a7113811d7f295 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.715454:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.715456:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.715458:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.715820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.719813:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.719838:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.719860:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.719885:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624622848 00000100:00100000:0.0:1769074478.719894:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624622848 00000100:00100000:0.0:1769074478.719928:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624623104 00000100:00100000:0.0:1769074478.719930:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624623104 00000100:00100000:0.0:1769074478.719941:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624623360 00000100:00100000:0.0:1769074478.719943:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624623360 00000100:00100000:0.0:1769074478.719957:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 73 00000100:00100000:0.0:1769074478.719966:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005661500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:32370:x1855007624622848:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.719985:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720045:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.720052:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005661500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:32370:x1855007624622848:12345-192.168.204.102@tcp:104: Request processed in 87us (249us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.720059:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 73 00000100:00100000:0.0:1769074478.720075:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 74 00000100:00100000:0.0:1769074478.720081:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005662d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33940:x1855007624623104:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.720089:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720114:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.720118:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005662d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33940:x1855007624623104:12345-192.168.204.102@tcp:104: Request processed in 39us (280us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.720124:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 74 00000100:00100000:0.0:1769074478.720136:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 75 00000100:00100000:0.0:1769074478.720143:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005663100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624623360:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.720151:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720176:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074478.720181:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005663100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6522:x1855007624623360:12345-192.168.204.102@tcp:104: Request processed in 39us (320us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.720185:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 75 00010000:00010000:0.0:1769074478.720199:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074478.720204:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720212:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006054c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.720217:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720256:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720263:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.720273:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720279:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054c00/0x68a7113811d7f256 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305b9a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720301:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074478.720305:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720310:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005727000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.720314:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720341:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 2/0,0 mode: --/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720347:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.720352:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 1/0,0 mode: --/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720357:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005727000/0x68a7113811d7f2bf lrc: 0/0,0 mode: --/PW res: [0x280000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305baba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720373:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.720376:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720382:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006055a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.720385:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720401:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720407:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.720413:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720420:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006055a00/0x68a7113811d7f280 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ba12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.720433:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.720435:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.720437:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.721900:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.722026:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074478.722041:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624623488 00000100:00100000:0.0:1769074478.722048:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624623488 00000100:00100000:0.0:1769074478.722089:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 76 00000100:00100000:0.0:1769074478.722097:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33944:x1855007624623488:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.722111:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1c00/0x68a7113811d7e6da lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059b02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.722178:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.722185:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33944:x1855007624623488:12345-192.168.204.102@tcp:104: Request processed in 89us (287us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.722192:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 76 00000100:00100000:1.0:1769074478.723725:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312192:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723852:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312704:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723878:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312960:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723940:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312064:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723968:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311936:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723981:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311808:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.723991:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311680:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724002:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311424:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724013:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612311040:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724046:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612310784:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724056:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612310656:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724095:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074478.724102:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312320:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724126:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312832:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724166:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612311552:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724176:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612311296:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.724185:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612310912:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.732498:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732550:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732579:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732595:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732602:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732610:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.732812:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624624000 00000100:00100000:1.0:1769074478.732824:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624624000 00000100:00100000:1.0:1769074478.732874:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624624128 00000100:00100000:1.0:1769074478.732879:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624624128 00000100:00100000:1.0:1769074478.732892:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624624256 00000100:00100000:1.0:1769074478.732897:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624624256 00000100:00100000:1.0:1769074478.732910:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624624640 00000100:00100000:1.0:1769074478.732913:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624624640 00000100:00100000:1.0:1769074478.732928:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624624768 00000100:00100000:1.0:1769074478.732931:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624624768 00000100:00100000:1.0:1769074478.732979:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624625024 00000100:00100000:1.0:1769074478.732985:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624625024 00000100:00100000:1.0:1769074478.733006:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 77 00000100:00100000:1.0:1769074478.733017:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+9:33934:x1855007624624000:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733036:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d7f2e9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bb2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733104:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.733114:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+9:33934:x1855007624624000:12345-192.168.204.102@tcp:104: Request processed in 98us (625us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733126:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 77 00000100:00100000:1.0:1769074478.733152:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 78 00000100:00100000:1.0:1769074478.733160:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+8:33937:x1855007624624128:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733170:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d7f313 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bb9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733212:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.733221:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+8:33937:x1855007624624128:12345-192.168.204.102@tcp:104: Request processed in 61us (670us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733242:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 78 00000100:00100000:1.0:1769074478.733274:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 79 00000100:00100000:1.0:1769074478.733290:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33938:x1855007624624256:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733302:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a600/0x68a7113811d7f33d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bc0a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733345:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.733362:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33938:x1855007624624256:12345-192.168.204.102@tcp:104: Request processed in 72us (778us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733380:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 79 00000100:00100000:1.0:1769074478.733400:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 80 00000100:00100000:1.0:1769074478.733420:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:6519:x1855007624624640:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733445:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d7f367 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bc7a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733485:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.733493:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:6519:x1855007624624640:12345-192.168.204.102@tcp:104: Request processed in 79us (897us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733503:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 80 00000100:00100000:1.0:1769074478.733526:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 81 00000100:00100000:1.0:1769074478.733537:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8372:x1855007624624768:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733558:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cada00/0x68a7113811d7e6ef lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059b3a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733620:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.733630:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8372:x1855007624624768:12345-192.168.204.102@tcp:104: Request processed in 93us (1023us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733701:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 81 00000100:00100000:1.0:1769074478.733728:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 82 00000100:00100000:1.0:1769074478.733741:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624625024:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.733759:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060b8000/0x68a7113811d7f391 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bcea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.733793:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.733821:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006042680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624625024:12345-192.168.204.102@tcp:104: Request processed in 82us (1205us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.733845:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 82 00000100:00100000:0.0:1769074478.735579:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.735734:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.735775:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.735891:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074478.735910:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624625664 00000100:00100000:0.0:1769074478.735919:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624625664 00000100:00100000:0.0:1769074478.735972:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 83 00000100:00100000:0.0:1769074478.735982:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1002bb4450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624625664:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.735998:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006054a00/0x68a7113811d7f2d4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305baf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.736059:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.736065:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1002bb4450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624625664:12345-192.168.204.102@tcp:104: Request processed in 84us (493us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.736072:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 83 00000100:00100000:1.0:1769074478.743144:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312192:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.747685:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.749557:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312320:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.749575:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312832:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.749590:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312704:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.749601:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612312960:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.749718:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624626048 00000100:00100000:0.0:1769074478.749727:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624626048 00000100:00100000:0.0:1769074478.749803:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 84 00000100:00100000:0.0:1769074478.749816:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006043100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33935:x1855007624626048:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.749838:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005726000/0x68a7113811d7f2fe lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bb62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.749951:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.749966:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006043100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33935:x1855007624626048:12345-192.168.204.102@tcp:104: Request processed in 145us (2319us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.749976:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 84 00000100:00100000:1.0:1769074478.757448:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.757516:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.757557:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624626304 00000100:00100000:1.0:1769074478.757569:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624626304 00000100:00100000:1.0:1769074478.757627:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624626560 00000100:00100000:1.0:1769074478.757631:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624626560 00000100:00100000:1.0:1769074478.757706:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 85 00000100:00100000:1.0:1769074478.757718:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624626304:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.757741:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.757835:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.757842:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624626304:12345-192.168.204.102@tcp:104: Request processed in 127us (414us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.757849:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 85 00000100:00100000:1.0:1769074478.757870:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 86 00000100:00100000:1.0:1769074478.757877:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624626560:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.757886:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.757921:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.757928:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624626560:12345-192.168.204.102@tcp:104: Request processed in 51us (412us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.757952:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 86 00010000:00010000:1.0:1769074478.757975:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.757982:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.757990:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.757998:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758051:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758059:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.758084:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758093:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d7f352 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bc42 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758121:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.758125:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758131:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d2000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.758135:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758157:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 2/0,0 mode: --/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758174:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.758181:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 1/0,0 mode: --/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758188:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2000/0x68a7113811d7f3bb lrc: 0/0,0 mode: --/PW res: [0x280000400:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd5a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758205:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.758208:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.758210:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.758230:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314624:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.758299:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.758307:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314752:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.758473:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.758493:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624626816 00000100:00100000:1.0:1769074478.758499:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624626816 00000100:00100000:1.0:1769074478.758521:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 87 00000100:00100000:1.0:1769074478.758528:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006043480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6519:x1855007624626816:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.758537:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758570:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.758576:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006043480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6519:x1855007624626816:12345-192.168.204.102@tcp:104: Request processed in 48us (105us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.758582:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 87 00010000:00010000:1.0:1769074478.758598:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.758608:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758615:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100601c400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.758620:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758666:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758674:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.758686:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.758706:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c400/0x68a7113811d7f3e5 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bdca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.758732:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314880:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.758767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.758770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.758772:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.758806:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.758858:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.758950:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624627456 00000100:00100000:1.0:1769074478.758955:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624627456 00000100:00100000:1.0:1769074478.758971:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624627584 00000100:00100000:1.0:1769074478.758974:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624627584 00000100:00100000:1.0:1769074478.758986:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 88 00000100:00100000:1.0:1769074478.758992:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:32370:x1855007624627456:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.759008:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759044:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.759050:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:32370:x1855007624627456:12345-192.168.204.102@tcp:104: Request processed in 58us (244us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.759057:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 88 00000100:00100000:1.0:1769074478.759070:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 89 00000100:00100000:1.0:1769074478.759076:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33936:x1855007624627584:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.759085:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759112:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.759118:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006041f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33936:x1855007624627584:12345-192.168.204.102@tcp:104: Request processed in 41us (260us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.759124:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 89 00010000:00010000:1.0:1769074478.759138:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.759143:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759152:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005654c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.759156:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759179:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759198:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.759210:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759219:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005654c00/0x68a7113811d7f37c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bcb2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759235:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.759239:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759245:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d3a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.759249:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759268:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759274:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.759279:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759287:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d3a00/0x68a7113811d7f439 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305beaa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.759302:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.759304:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.759310:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.759323:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315136:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.759355:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.759362:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315008:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.769144:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.769190:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.769238:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.769808:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.769841:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624628224 00000100:00100000:1.0:1769074478.769854:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624628224 00000100:00100000:1.0:1769074478.769919:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 90 00000100:00100000:1.0:1769074478.769931:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1002bb4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33937:x1855007624628224:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.769952:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.770046:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.770054:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1002bb4050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33937:x1855007624628224:12345-192.168.204.102@tcp:104: Request processed in 126us (924us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.770062:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 90 00010000:00010000:1.0:1769074478.770107:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.770115:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.770124:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d3400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.770131:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.770185:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.770217:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.770244:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.770252:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3400/0x68a7113811d7f3a6 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bd22 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074478.770290:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074478.770309:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315264:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.770373:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.770375:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.770377:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.771585:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.771724:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.771758:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.771778:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624628608 00000100:00100000:0.0:1769074478.771782:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624628608 00000100:00100000:0.0:1769074478.771801:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624628736 00000100:00100000:0.0:1769074478.771803:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624628736 00000100:00100000:0.0:1769074478.771810:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624628864 00000100:00100000:0.0:1769074478.771812:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624628864 00000100:00100000:0.0:1769074478.771822:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 91 00000100:00100000:0.0:1769074478.771828:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005662a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33938:x1855007624628608:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.771836:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.771868:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.771874:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005662a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33938:x1855007624628608:12345-192.168.204.102@tcp:104: Request processed in 46us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.771881:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 91 00000100:00100000:0.0:1769074478.771895:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 92 00000100:00100000:0.0:1769074478.771902:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005663800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33943:x1855007624628736:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.771910:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.771939:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074478.771944:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005663800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33943:x1855007624628736:12345-192.168.204.102@tcp:104: Request processed in 43us (221us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.771953:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 92 00000100:00100000:0.0:1769074478.771976:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 93 00000100:00100000:0.0:1769074478.771982:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005660700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624628864:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.771989:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772016:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074478.772021:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005660700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624628864:12345-192.168.204.102@tcp:104: Request processed in 39us (263us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.772027:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 93 00010000:00010000:0.0:1769074478.772043:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074478.772048:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772055:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.772059:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772082:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772089:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.772105:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772112:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d7f424 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x21a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.772144:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315648:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.772182:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074478.772186:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772193:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005cad200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.772196:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772212:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772218:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.772227:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772233:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cad200/0x68a7113811d7e704 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059b72 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.772253:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612315776:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.772284:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.772288:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772295:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.772298:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772427:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 2/0,0 mode: --/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772434:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.772447:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 1/0,0 mode: --/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772454:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b8200/0x68a7113811d7f3fa lrc: 0/0,0 mode: --/PW res: [0x240000400:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305be02 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.772475:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315904:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.772504:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.772506:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.772508:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.772556:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074478.772574:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624629120 00000100:00100000:0.0:1769074478.772579:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624629120 00000100:00100000:0.0:1769074478.772599:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 94 00000100:00100000:0.0:1769074478.772604:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005662680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624629120:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074478.772612:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772637:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074478.772681:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005662680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624629120:12345-192.168.204.102@tcp:104: Request processed in 76us (125us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074478.772701:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 94 00010000:00010000:0.0:1769074478.772716:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.772720:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772727:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b9c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.772731:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772749:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772754:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.772764:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.772772:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d7f44e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x21b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bee2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.772790:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612316032:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.772820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.772822:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.772823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.772934:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314624:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.773064:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.774212:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774227:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774248:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774268:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774299:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774338:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774355:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774442:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.774476:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074478.774490:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624630144 00000100:00100000:1.0:1769074478.774497:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624630144 00000100:00100000:1.0:1769074478.774532:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624630528 00000100:00100000:1.0:1769074478.774533:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624630528 00000100:00100000:1.0:1769074478.774539:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624630656 00000100:00100000:1.0:1769074478.774557:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624630656 00000100:00100000:1.0:1769074478.774570:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 95 00000100:00100000:1.0:1769074478.774576:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6519:x1855007624630144:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.774585:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774624:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.774628:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6519:x1855007624630144:12345-192.168.204.102@tcp:104: Request processed in 53us (418us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.774633:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 95 00000100:00100000:1.0:1769074478.774661:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 96 00000100:00100000:1.0:1769074478.774665:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:32370:x1855007624630528:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.774671:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774688:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.774703:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:32370:x1855007624630528:12345-192.168.204.102@tcp:104: Request processed in 26us (464us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.774709:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 96 00010000:00010000:1.0:1769074478.774723:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074478.774733:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774741:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060e1a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.774746:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774784:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.774799:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315008:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774801:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074478.774811:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315136:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774820:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.774821:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314880:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774828:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060e1a00/0x68a7113811d7f463 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bf1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.774830:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612314752:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774850:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074478.774853:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774860:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006133000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074478.774864:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074478.774873:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612316160:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774881:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774887:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074478.774894:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774900:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006133000/0x68a7113811d7f50b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c0da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.774912:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.774913:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:1.0:1769074478.774914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.774919:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612316288:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074478.774921:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074478.774933:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 97 00000100:00100000:1.0:1769074478.774940:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33940:x1855007624630656:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.774948:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1800/0x68a7113811d7f478 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bf52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.774952:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074478.776192:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.776198:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f7c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33940:x1855007624630656:12345-192.168.204.102@tcp:104: Request processed in 1259us (1950us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.776205:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 97 00000100:00100000:1.0:1769074478.782222:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315264:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074478.792798:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.792831:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.792928:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.792962:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624631808 00000100:00100000:1.0:1769074478.792972:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624631808 00000100:00100000:1.0:1769074478.793018:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624631936 00000100:00100000:1.0:1769074478.793021:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624631936 00000100:00100000:1.0:1769074478.793034:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624631424 00000100:00100000:1.0:1769074478.793036:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624631424 00000100:00100000:1.0:1769074478.793051:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 98 00000100:00100000:1.0:1769074478.793060:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006043100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6522:x1855007624631808:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.793075:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100603ec00/0x68a7113811d7f4f6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c0a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.793153:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.793161:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006043100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6522:x1855007624631808:12345-192.168.204.102@tcp:104: Request processed in 101us (366us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.793187:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 98 00000100:00100000:1.0:1769074478.793210:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 99 00000100:00100000:1.0:1769074478.793217:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:32370:x1855007624631936:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.793225:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.793271:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074478.793277:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:32370:x1855007624631936:12345-192.168.204.102@tcp:104: Request processed in 61us (445us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.793283:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 99 00000100:00100000:1.0:1769074478.793295:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 100 00000100:00100000:1.0:1769074478.793301:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33937:x1855007624631424:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.793308:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.793345:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074478.793350:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006040700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33937:x1855007624631424:12345-192.168.204.102@tcp:104: Request processed in 49us (422us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.793355:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 100 00000100:00100000:0.0:1769074478.794731:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315648:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.794771:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612316288:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.794781:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612316160:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.794792:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612315904:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.794832:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612316032:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.794843:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612315776:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074478.797581:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074478.797595:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797609:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060e0000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.797618:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797723:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 2/0,0 mode: --/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797731:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.797766:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 1/0,0 mode: --/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797775:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e0000/0x68a7113811d7f4cc lrc: 0/0,0 mode: --/PW res: [0x240000400:0x21e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c032 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797802:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074478.797806:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797812:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060e1c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074478.797816:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797833:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797840:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074478.797845:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797852:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1c00/0x68a7113811d7f4a2 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305bfc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074478.797871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.797873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074478.797874:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074478.797891:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612317824:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.797951:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074478.797957:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612317696:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.803527:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074478.803814:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624632192 00000100:00100000:1.0:1769074478.803829:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624632192 00000100:00100000:1.0:1769074478.803900:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 101 00000100:00100000:1.0:1769074478.803912:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1005661180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624632192:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074478.803936:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601d000/0x68a7113811d7f520 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c112 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074478.804072:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074478.804079:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1005661180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:14023:x1855007624632192:12345-192.168.204.102@tcp:104: Request processed in 170us (569us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074478.804088:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 101 00000100:00100000:0.0:1769074478.807605:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612317696:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074478.807618:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612317824:192.168.204.102@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1769074478.831960:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0xfd:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074478.831981:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074478.832020:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0xfd:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074478.832094:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xfd:0x0](ffff8b100485b248) 00000008:00010000:0.0:1769074478.832139:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 8 blocks 1 [cma]time: 1769074414 1769074414 0 by oinfo size 8 blocks 1 [cma]time 1769074414 1769074414 1769074475 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074478.832264:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074478.832290:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074478.832376:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80167 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.832386:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80167 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074478.832403:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80167 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074478.832430:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612318720:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074478.832521:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2840 going to sleep for 11 seconds 00010000:00010000:3.0:1769074479.017601:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.017612:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.017616:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.017619:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801b4 00000080:00010000:3.0:1769074479.017622:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801b4 from it ffffaa914211fd68 00010000:00010000:3.0:1769074479.017626:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.017634:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.017778:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074479.017785:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-24, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074479.017822:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.017871:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ee00/0x68a7113811d7f899 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e69d expref: -99 pid: 30758 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.017884:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x42:0x0]: losing layout lock 00010000:00010000:3.0:1769074479.017931:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ee00/0x68a7113811d7f899 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e69d expref: -99 pid: 30758 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.017939:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.017945:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074479.017948:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ee00/0x68a7113811d7f899 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x42:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e69d expref: -99 pid: 30758 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.018208:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801b4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061abb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074479.018233:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074479.018545:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037df800 x1855007612323200/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074479.018579:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323200:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074479.018816:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.025787:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323200:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074479.025872:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x42:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074479.025890:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074479.025926:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x42:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.026010:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x42:0x0](ffff8b1018197448) 00000008:00010000:3.0:1769074479.026053:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074479.026164:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.026188:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.027911:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.027931:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.027945:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.028095:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323328:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.028425:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.032760:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323328:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.032793:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032808:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032823:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032836:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032850:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032858:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032865:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.032877:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de300 x1855007612323328/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.032914:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.032929:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.032937:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074479.032991:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074479.032998:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801bb 00000080:00010000:1.0:1769074479.033006:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.033009:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801bb from it ffffaa914211fb58 00010000:00010000:1.0:1769074479.033012:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033022:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.033053:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.033070:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-25, flags=8 00000080:00200008:1.0:1769074479.033087:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-25:[0x200001b71:0x43:0x0](ffff8b1004809988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074479.033093:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x43:0x0](ffff8b1004809988),name=f5.replay-single-25 00000002:00010000:1.0:1769074479.033118:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x43:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.033129:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.033140:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.033147:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.033152:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x43:0x0](ffff8b1004809988) 00000080:00010000:1.0:1769074479.033160:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x43:0x0](ffff8b1004809988) for lock 0x68a7113811d7f8a0 00000080:00010000:1.0:1769074479.033165:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8a0 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074479.033168:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.033175:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.033199:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x43:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074479.033212:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x43:0x0] 00000008:00010000:1.0:1769074479.033238:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074479.033430:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2eb0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074479.033456:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.033468:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.033481:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074479.033485:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.033493:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.033505:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074479.033520:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x43:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074479.033525:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074479.033528:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.033539:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2eb0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.033543:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074479.033546:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2eb0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074479.033558:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x43:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.033600:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.033609:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.033617:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033627:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.033634:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.033785:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.033790:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801bb 00000080:00010000:1.0:1769074479.033795:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801bb from it ffffaa914211fc10 00010000:00010000:1.0:1769074479.033798:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033807:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.033817:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.033826:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-25, flags=0 00000080:00200000:1.0:1769074479.033831:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.033838:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.033845:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033868:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.033875:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.033880:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.033883:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801bb 00000080:00010000:1.0:1769074479.033887:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801bb from it ffffaa914211fd68 00010000:00010000:1.0:1769074479.033890:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033899:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.033907:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074479.033914:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-25, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074479.033941:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.033993:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.034006:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x43:0x0]: losing layout lock 00010000:00010000:1.0:1769074479.034048:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.034058:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.034064:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074479.034073:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fc00/0x68a7113811d7f8a0 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e6c0 expref: -99 pid: 30759 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.034096:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d801bb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061ae5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074479.034119:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074479.034130:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041c00 x1855007612323584/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074479.034160:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323584:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074479.034932:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.041404:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323584:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074479.041488:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x43:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074479.041507:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074479.041542:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x43:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.041613:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x43:0x0](ffff8b1004809988) 00000008:00010000:1.0:1769074479.041739:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.041841:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.041866:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.041942:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.041950:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.042187:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.042228:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323712:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.042559:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.045902:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323712:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.045938:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.045952:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.045979:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.045992:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046005:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046013:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046021:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.046034:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006043b80 x1855007612323712/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.046048:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046082:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.046090:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074479.046122:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074479.046130:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c2 00000080:00010000:1.0:1769074479.046136:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.046140:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c2 from it ffffaa914211fb58 00010000:00010000:1.0:1769074479.046143:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046151:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.046183:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.046198:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-26, flags=8 00000080:00200008:1.0:1769074479.046213:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-26:[0x200001b71:0x44:0x0](ffff8b100480f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074479.046219:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x44:0x0](ffff8b100480f448),name=f5.replay-single-26 00000002:00010000:1.0:1769074479.046254:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x44:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.046268:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.046280:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.046287:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.046291:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x44:0x0](ffff8b100480f448) 00000080:00010000:1.0:1769074479.046294:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x44:0x0](ffff8b100480f448) for lock 0x68a7113811d7f8a7 00000080:00010000:1.0:1769074479.046298:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8a7 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074479.046307:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.046324:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.046351:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x44:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074479.046365:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x44:0x0] 00000008:00010000:1.0:1769074479.046390:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074479.046394:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f28b0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074479.046414:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.046425:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.046438:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074479.046442:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.046450:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.046461:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074479.046475:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x44:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074479.046480:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074479.046484:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.046487:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f28b0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.046492:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074479.046500:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f28b0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074479.046512:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x44:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.046550:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.046557:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.046564:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046573:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.046580:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.046584:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.046586:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c2 00000080:00010000:1.0:1769074479.046590:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c2 from it ffffaa914211fc10 00010000:00010000:1.0:1769074479.046593:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046599:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.046607:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.046614:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-26, flags=0 00000080:00200000:1.0:1769074479.046618:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.046624:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.046629:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046635:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.046712:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.046717:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.046719:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c2 00000080:00010000:1.0:1769074479.046723:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c2 from it ffffaa914211fd68 00010000:00010000:1.0:1769074479.046726:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046733:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.046741:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074479.046747:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-26, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074479.046775:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046820:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.046833:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x44:0x0]: losing layout lock 00010000:00010000:1.0:1769074479.046877:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.046884:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.046889:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074479.046892:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602ca00/0x68a7113811d7f8a7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x44:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e6e3 expref: -99 pid: 30760 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.046926:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801c2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b0f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074479.046947:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074479.046958:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006043b80 x1855007612323968/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074479.046985:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323968:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074479.047075:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.053460:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006043b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612323968:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074479.053548:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x44:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074479.053568:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074479.053608:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x44:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.053720:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x44:0x0](ffff8b100480f448) 00000008:00010000:1.0:1769074479.053765:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.053868:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.053894:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.053975:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.053983:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.053992:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.054031:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324096:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.054098:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.059766:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324096:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.059810:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059826:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059841:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059855:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059869:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059878:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059887:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.059899:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006042300 x1855007612324096/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.059916:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.059952:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.059975:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074479.060014:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074479.060022:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c9 00000080:00010000:1.0:1769074479.060029:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.060033:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c9 from it ffffaa914211fb58 00010000:00010000:1.0:1769074479.060037:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060047:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.060081:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.060097:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-27, flags=8 00000080:00200008:1.0:1769074479.060113:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-27:[0x200001b71:0x45:0x0](ffff8b100480ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074479.060120:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x45:0x0](ffff8b100480ba88),name=f5.replay-single-27 00000002:00010000:1.0:1769074479.060150:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x45:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.060163:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.060175:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.060183:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.060187:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x45:0x0](ffff8b100480ba88) 00000080:00010000:1.0:1769074479.060190:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x45:0x0](ffff8b100480ba88) for lock 0x68a7113811d7f8ae 00000080:00010000:1.0:1769074479.060195:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8ae from it ffffaa914211fcb8 00010000:00010000:1.0:1769074479.060204:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.060211:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.060237:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x45:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074479.060252:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x45:0x0] 00000008:00010000:1.0:1769074479.060278:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074479.060283:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f28b0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074479.060303:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.060313:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.060326:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074479.060330:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.060339:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.060351:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074479.060368:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x45:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074479.060373:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074479.060376:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.060379:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f28b0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.060384:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074479.060393:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f28b0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074479.060406:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x45:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.060447:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.060455:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.060461:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060469:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.060475:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.060478:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.060481:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c9 00000080:00010000:1.0:1769074479.060484:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c9 from it ffffaa914211fc10 00010000:00010000:1.0:1769074479.060487:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060494:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.060502:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.060509:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-27, flags=0 00000080:00200000:1.0:1769074479.060514:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.060519:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.060525:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060532:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.060548:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.060552:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.060555:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801c9 00000080:00010000:1.0:1769074479.060560:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801c9 from it ffffaa914211fd68 00010000:00010000:1.0:1769074479.060563:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060569:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.060577:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074479.060583:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-27, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074479.060613:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060711:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.060724:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x45:0x0]: losing layout lock 00010000:00010000:1.0:1769074479.060766:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.060772:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.060778:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074479.060781:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f400/0x68a7113811d7f8ae lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x45:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e706 expref: -99 pid: 30761 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.060813:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d801c9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b39 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074479.060835:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074479.060845:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042300 x1855007612324352/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074479.060872:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324352:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074479.060984:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.066006:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324352:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074479.066087:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x45:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074479.066106:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074479.066142:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x45:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.066208:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x45:0x0](ffff8b100480ba88) 00000008:00010000:1.0:1769074479.066262:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.066540:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.066563:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.066673:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.066683:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.066704:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.066748:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324480:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.066833:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.071771:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324480:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.071807:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071821:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071833:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071846:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071859:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071866:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071873:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.071884:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006041180 x1855007612324480/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.071899:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.071934:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.071943:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074479.071989:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074479.071997:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801d0 00000080:00010000:1.0:1769074479.072005:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.072008:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801d0 from it ffffaa914211fb58 00010000:00010000:1.0:1769074479.072011:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.072020:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.072050:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.072065:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-28, flags=8 00000080:00200008:1.0:1769074479.072081:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-28:[0x200001b71:0x46:0x0](ffff8b10048080c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074479.072087:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x46:0x0](ffff8b10048080c8),name=f5.replay-single-28 00000002:00010000:1.0:1769074479.072113:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x46:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.072124:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.072134:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.072140:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.072145:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x46:0x0](ffff8b10048080c8) 00000080:00010000:1.0:1769074479.072148:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x46:0x0](ffff8b10048080c8) for lock 0x68a7113811d7f8b5 00000080:00010000:1.0:1769074479.072152:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8b5 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074479.072160:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.072166:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.072190:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x46:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074479.072204:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x46:0x0] 00000008:00010000:1.0:1769074479.072232:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074479.072237:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f28b0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074479.072268:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.072278:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.072292:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074479.072296:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.072304:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.072634:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074479.072677:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x46:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074479.072683:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074479.072697:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.072701:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f28b0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.072707:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074479.072716:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f28b0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074479.072731:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x46:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.072774:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.072783:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.072792:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.072801:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.072807:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.072812:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.072815:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801d0 00000080:00010000:1.0:1769074479.072820:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801d0 from it ffffaa914211fc10 00010000:00010000:1.0:1769074479.072823:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.072831:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.072839:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.072848:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-28, flags=0 00000080:00200000:1.0:1769074479.072853:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.072858:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.072864:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.072872:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.072889:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.072893:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.072896:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801d0 00000080:00010000:1.0:1769074479.072900:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801d0 from it ffffaa914211fd68 00010000:00010000:1.0:1769074479.072904:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.072912:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.072920:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074479.072926:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-28, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074479.072954:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.073102:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.073115:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x46:0x0]: losing layout lock 00010000:00010000:1.0:1769074479.073159:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.073168:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.073174:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074479.073178:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5200/0x68a7113811d7f8b5 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x46:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e729 expref: -99 pid: 30762 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.073211:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9a00/0x68a7113811d801d0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061b63 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074479.073232:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074479.073475:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041180 x1855007612324736/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074479.073508:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324736:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074479.073618:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.078946:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612324736:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074479.079208:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x46:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074479.079228:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074479.079402:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x46:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.079476:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x46:0x0](ffff8b10048080c8) 00000008:00010000:1.0:1769074479.079525:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074466 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.079636:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000080:00200008:3.0:1769074479.150607:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-32:[0x200001b71:0x4a:0x0](ffff8b100480d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074479.150622:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x4a:0x0](ffff8b100480d348),name=f5.replay-single-32 00000002:00010000:3.0:1769074479.150681:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x4a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.150709:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.150731:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.150738:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.150747:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x4a:0x0](ffff8b100480d348) 00000080:00010000:3.0:1769074479.150751:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x4a:0x0](ffff8b100480d348) for lock 0x68a7113811d7f8d1 00000080:00010000:3.0:1769074479.150757:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8d1 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074479.150761:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.150770:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.150801:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x4a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074479.150812:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x4a:0x0] 00000008:00010000:3.0:1769074479.150832:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074479.150835:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76630, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074479.150853:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.150864:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.150877:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074479.150881:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.150889:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.150899:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074479.150911:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x4a:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074479.150917:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:3.0:1769074479.150920:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.150925:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76630 (ffff8b10056851b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.150929:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074479.150932:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76630 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074479.150943:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x4a:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.150984:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.150993:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.151011:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151020:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.151025:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.151029:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.151031:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801f3 00000080:00010000:3.0:1769074479.151035:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801f3 from it ffffaa914211fc10 00010000:00010000:3.0:1769074479.151038:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151044:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.151052:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.151061:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-32, flags=0 00000080:00200000:3.0:1769074479.151076:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.151082:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.151088:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151095:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.151101:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.151104:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.151107:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801f3 00000080:00010000:3.0:1769074479.151110:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801f3 from it ffffaa914211fd68 00010000:00010000:3.0:1769074479.151112:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151119:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.151126:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074479.151131:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-32, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074479.151329:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151366:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.151377:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x4a:0x0]: losing layout lock 00010000:00010000:3.0:1769074479.151414:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.151425:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.151430:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074479.151433:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602da00/0x68a7113811d7f8d1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x4a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e7b5 expref: -99 pid: 30766 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.151538:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d801f3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061c35 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074479.151561:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074479.151573:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037df100 x1855007612326656/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074479.151599:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612326656:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074479.151770:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.161784:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612326656:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074479.161850:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x4a:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074479.161863:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074479.161889:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x4a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.161940:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x4a:0x0](ffff8b100480d348) 00000008:00010000:3.0:1769074479.161971:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074467 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074479.162050:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.162080:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.162142:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.162151:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.162160:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074479.162179:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612326784:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074479.162249:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.177719:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612326784:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074479.177751:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177765:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177776:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177787:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177798:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177806:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177813:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074479.177824:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037df480 x1855007612326784/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074479.177836:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177848:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074479.177855:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074479.177882:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074479.177888:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801fa 00000080:00010000:3.0:1769074479.177894:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.177897:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801fa from it ffffaa914211fb58 00010000:00010000:3.0:1769074479.177901:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.177909:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.177932:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.177943:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-33, flags=8 00000080:00200008:3.0:1769074479.177954:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-33:[0x200001b71:0x4b:0x0](ffff8b100480cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074479.177959:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x4b:0x0](ffff8b100480cb08),name=f5.replay-single-33 00000002:00010000:3.0:1769074479.177978:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x4b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.177987:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.177997:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.178004:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.178008:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x4b:0x0](ffff8b100480cb08) 00000080:00010000:3.0:1769074479.178011:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x4b:0x0](ffff8b100480cb08) for lock 0x68a7113811d7f8d8 00000080:00010000:3.0:1769074479.178015:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f8d8 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074479.178018:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.178025:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.178046:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x4b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074479.178058:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x4b:0x0] 00000008:00010000:3.0:1769074479.178089:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074479.178093:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76530, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074479.178110:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.178119:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.178130:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074479.178134:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.178141:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.178154:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074479.178165:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x4b:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074479.178168:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:3.0:1769074479.178171:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.178174:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76530 (ffff8b10056851b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.178178:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074479.178180:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76530 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074479.178189:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x4b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.178224:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.178232:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.178239:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178246:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.178252:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.178255:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.178257:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801fa 00000080:00010000:3.0:1769074479.178261:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801fa from it ffffaa914211fc10 00010000:00010000:3.0:1769074479.178264:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178270:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.178278:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.178286:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-33, flags=0 00000080:00200000:3.0:1769074479.178290:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.178295:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.178300:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178307:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.178313:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.178316:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.178319:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d801fa 00000080:00010000:3.0:1769074479.178322:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d801fa from it ffffaa914211fd68 00010000:00010000:3.0:1769074479.178325:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178332:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.178339:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074479.178344:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-33, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074479.178363:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178398:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.178407:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x4b:0x0]: losing layout lock 00010000:00010000:3.0:1769074479.178439:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.178600:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.178605:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074479.178609:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5c00/0x68a7113811d7f8d8 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e7d8 expref: -99 pid: 30767 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.178628:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d801fa lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061c5f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074479.178680:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074479.178698:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037df480 x1855007612327040/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074479.178722:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612327040:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074479.178798:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.204576:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612327040:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.204812:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x4b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074479.204828:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.204855:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x4b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.204905:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x4b:0x0](ffff8b100480cb08) 00000008:00010000:0.0:1769074479.204935:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 7 blocks 1 [cma]time 1769074407 1769074407 1769074467 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.205020:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.205038:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.205092:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d80201 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.205100:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d80201 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.205111:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d80201 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.205130:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612327168:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.205222:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2000 going to sleep for 11 seconds 00000080:00200000:0.0:1769074479.335204:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.335215:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8022b 00000080:00010000:0.0:1769074479.335232:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.335236:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8022b from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.335240:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335249:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.335288:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.335304:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-4, flags=8 00000080:00200008:0.0:1769074479.335319:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-4:[0x200001b71:0x2e:0x0](ffff8b1018190908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.335337:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x2e:0x0](ffff8b1018190908),name=f5.replay-single-4 00000002:00010000:0.0:1769074479.335361:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x2e:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.335372:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.335384:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.335392:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.335397:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x2e:0x0](ffff8b1018190908) 00000080:00010000:0.0:1769074479.335402:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x2e:0x0](ffff8b1018190908) for lock 0x68a7113811d7f7d5 00000080:00010000:0.0:1769074479.335406:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7d5 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.335408:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.335415:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.335440:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x2e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.335454:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x2e:0x0] 00000008:00010000:0.0:1769074479.335478:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.335483:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d130, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.335505:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.335516:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.335530:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.335534:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.335542:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.335554:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.335570:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x2e:0x0] updating i_size 6 i_blocks 1 00000020:00010000:0.0:1769074479.335574:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.335576:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.335580:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d130 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.335583:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.335585:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d130 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.335596:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x2e:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.335634:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.335665:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.335671:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335677:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.335691:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.335695:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.335698:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8022b 00000080:00010000:0.0:1769074479.335702:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8022b from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.335706:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335714:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.335723:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.335732:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-4, flags=0 00000080:00200000:0.0:1769074479.335736:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.335742:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.335747:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335761:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.335767:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.335771:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.335774:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8022b 00000080:00010000:0.0:1769074479.335778:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8022b from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.335781:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335788:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.335795:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.335800:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-4, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.335829:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335871:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.335889:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x2e:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.335929:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.335938:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.335943:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.335946:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642ec00/0x68a7113811d7f7d5 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x2e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e301 expref: -99 pid: 30736 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.335969:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d8022b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061d85 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.335991:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.336001:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005662d80 x1855007612329728/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.336028:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612329728:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.336137:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a23c0 going to sleep for 11 seconds 00000080:00200000:0.0:1769074479.354902:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.354928:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.355006:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.355016:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.355029:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.355059:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330240:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.355393:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.362890:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330240:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.362927:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.362943:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363067:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363083:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363099:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363109:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363117:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.363130:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005661880 x1855007612330240/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.363147:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363164:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.363172:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.363214:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.363232:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80239 00000080:00010000:0.0:1769074479.363246:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.363250:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80239 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.363254:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.363265:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.363587:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.363602:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-41, flags=8 00000080:00200008:0.0:1769074479.363620:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-41:[0x200001b71:0x53:0x0](ffff8b10048100c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.363627:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x53:0x0](ffff8b10048100c8),name=f5.replay-single-41 00000002:00010000:0.0:1769074479.363691:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x53:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.363705:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.363717:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.363725:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.363731:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x53:0x0](ffff8b10048100c8) 00000080:00010000:0.0:1769074479.363734:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x53:0x0](ffff8b10048100c8) for lock 0x68a7113811d7f910 00000080:00010000:0.0:1769074479.363739:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f910 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.363742:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.363750:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.363777:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x53:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.363791:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x53:0x0] 00000008:00010000:0.0:1769074479.363817:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.363821:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.363845:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.363856:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.363871:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.363876:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.363884:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.363896:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.363910:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x53:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.363915:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.363919:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.363922:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.363927:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.363930:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.363943:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x53:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.364087:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.364096:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.364104:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364114:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.364121:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.364126:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.364129:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80239 00000080:00010000:0.0:1769074479.364133:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80239 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.364137:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364145:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.364154:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.364163:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-41, flags=0 00000080:00200000:0.0:1769074479.364168:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.364174:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.364180:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364189:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.364196:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.364200:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.364203:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80239 00000080:00010000:0.0:1769074479.364207:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80239 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.364210:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364218:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.364244:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.364252:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-41, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.364573:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364619:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.364632:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x53:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.364743:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.364752:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.364758:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.364761:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642fe00/0x68a7113811d7f910 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e8f0 expref: -99 pid: 30775 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.364787:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80239 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061dd9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.364812:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.364824:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005661880 x1855007612330496/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.364855:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330496:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.365131:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.370122:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330496:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.370212:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x53:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.370231:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.370270:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x53:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.370351:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x53:0x0](ffff8b10048100c8) 00000008:00010000:0.0:1769074479.370477:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074467 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.370616:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.370676:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.370761:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.370772:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.370783:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.370808:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330624:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.370887:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.375568:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330624:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.375599:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375611:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375622:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375633:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375677:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375695:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375703:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.375716:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660000 x1855007612330624/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.375733:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375747:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.375753:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.375787:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.375792:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80240 00000080:00010000:0.0:1769074479.375797:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.375800:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80240 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.375802:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.375809:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.375837:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.375850:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-42, flags=8 00000080:00200008:0.0:1769074479.375866:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-42:[0x200001b71:0x54:0x0](ffff8b101817cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.375871:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x54:0x0](ffff8b101817cb08),name=f5.replay-single-42 00000002:00010000:0.0:1769074479.375891:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x54:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.375900:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.375909:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.375913:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.375915:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x54:0x0](ffff8b101817cb08) 00000080:00010000:0.0:1769074479.375917:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x54:0x0](ffff8b101817cb08) for lock 0x68a7113811d7f917 00000080:00010000:0.0:1769074479.375920:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f917 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.375921:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.375926:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.375949:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x54:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.375961:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x54:0x0] 00000008:00010000:0.0:1769074479.375986:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.375991:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.376012:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.376029:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.376043:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.376047:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.376055:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.376065:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.376081:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x54:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.376086:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.376089:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.376093:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.376097:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.376100:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.376112:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x54:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.376143:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.376148:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.376152:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376158:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.376162:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.376166:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.376168:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80240 00000080:00010000:0.0:1769074479.376170:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80240 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.376172:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376177:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.376181:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.376187:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-42, flags=0 00000080:00200000:0.0:1769074479.376191:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.376194:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.376197:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376201:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.376205:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.376207:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.376208:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80240 00000080:00010000:0.0:1769074479.376209:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80240 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.376211:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376215:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.376218:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.376225:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-42, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.376250:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376303:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.376314:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x54:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.376884:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.376897:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.376904:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.376907:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601c600/0x68a7113811d7f917 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e913 expref: -99 pid: 30776 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.376932:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80240 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e03 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.376955:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.377088:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005660000 x1855007612330880/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.377117:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330880:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.378061:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.385778:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612330880:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.385870:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x54:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.385889:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.385927:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x54:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.386122:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x54:0x0](ffff8b101817cb08) 00000008:00010000:0.0:1769074479.386165:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074467 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.386283:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.386308:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.386384:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.386394:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.386405:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.386437:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331008:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.387435:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.390383:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331008:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.390420:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390435:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390450:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390464:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390479:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390487:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390495:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.390507:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005663b80 x1855007612331008/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.390523:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390538:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.390546:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.390583:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.390591:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80247 00000080:00010000:0.0:1769074479.390597:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.390601:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80247 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.390605:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.390615:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.390750:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.390767:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-43, flags=8 00000080:00200008:0.0:1769074479.390781:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-43:[0x200001b71:0x55:0x0](ffff8b10048121c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.390788:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x55:0x0](ffff8b10048121c8),name=f5.replay-single-43 00000002:00010000:0.0:1769074479.390813:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x55:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.390824:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.390837:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.390844:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.390849:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x55:0x0](ffff8b10048121c8) 00000080:00010000:0.0:1769074479.390852:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x55:0x0](ffff8b10048121c8) for lock 0x68a7113811d7f91e 00000080:00010000:0.0:1769074479.390857:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f91e from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.390860:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.390868:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.390893:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x55:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.390908:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x55:0x0] 00000008:00010000:0.0:1769074479.390933:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.390937:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.390956:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.390993:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.391020:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.391026:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.391035:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.391046:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.391061:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x55:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.391066:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.391070:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.391073:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.391078:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.391082:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.391095:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x55:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.391134:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.391141:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.391149:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391157:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.391165:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.391169:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.391172:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80247 00000080:00010000:0.0:1769074479.391178:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80247 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.391181:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391190:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.391197:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.391204:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-43, flags=0 00000080:00200000:0.0:1769074479.391208:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.391213:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.391218:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391440:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.391448:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.391453:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.391456:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80247 00000080:00010000:0.0:1769074479.391461:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80247 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.391464:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391472:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.391482:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.391488:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-43, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.391521:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391564:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.391576:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x55:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.391620:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.391629:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.391636:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.391734:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100601d600/0x68a7113811d7f91e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e936 expref: -99 pid: 30777 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.391756:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80247 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e2d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.391778:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.391789:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005663b80 x1855007612331264/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.391819:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331264:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.391969:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.396414:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331264:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.396523:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x55:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.396544:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.396582:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x55:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.396723:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x55:0x0](ffff8b10048121c8) 00000008:00010000:0.0:1769074479.396784:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074467 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.396891:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.396916:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.396987:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.396997:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.397008:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.397033:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331392:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.397106:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.404087:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331392:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.404121:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404137:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404151:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404165:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404181:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404190:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404199:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.404210:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660380 x1855007612331392/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.404238:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404255:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.404263:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.404306:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.404315:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8024e 00000080:00010000:0.0:1769074479.404321:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.404325:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8024e from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.404329:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404339:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.404372:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.404388:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-44, flags=8 00000080:00200008:0.0:1769074479.404404:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-44:[0x200001b71:0x56:0x0](ffff8b1004813248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.404410:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x56:0x0](ffff8b1004813248),name=f5.replay-single-44 00000002:00010000:0.0:1769074479.404435:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x56:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.404446:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.404459:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.404465:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.404470:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x56:0x0](ffff8b1004813248) 00000080:00010000:0.0:1769074479.404472:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x56:0x0](ffff8b1004813248) for lock 0x68a7113811d7f925 00000080:00010000:0.0:1769074479.404477:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f925 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.404480:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.404486:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.404511:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x56:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.404525:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x56:0x0] 00000008:00010000:0.0:1769074479.404550:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.404554:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.404574:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.404585:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.404604:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.404609:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.404618:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.404629:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.404674:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x56:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.404711:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.404715:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.404718:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.404722:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.404725:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.404739:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x56:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.404781:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.404795:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.404804:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404812:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.404818:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.404823:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.404826:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8024e 00000080:00010000:0.0:1769074479.404831:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8024e from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.404835:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404842:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.404851:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.404860:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-44, flags=0 00000080:00200000:0.0:1769074479.404864:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.404870:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.404875:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404884:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.404890:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.404893:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.404897:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8024e 00000080:00010000:0.0:1769074479.404900:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8024e from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.404903:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404909:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.404917:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.404923:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-44, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.404952:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.404996:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.405008:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x56:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.405048:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.405055:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.405061:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.405065:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100642f000/0x68a7113811d7f925 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e959 expref: -99 pid: 30778 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.405086:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8024e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061e57 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.405109:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.405119:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005660380 x1855007612331648/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.405147:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331648:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.405259:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.429848:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612331648:192.168.204.102@tcp:36:rm.0 00000002:00010000:0.0:1769074479.511136:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.511155:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.511173:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.511181:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.511191:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.511196:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80271 00000080:00010000:0.0:1769074479.511201:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80271 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.511204:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.511213:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.511230:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.511594:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-49, flags=0 00000080:00200000:0.0:1769074479.511603:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.511612:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.511620:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.511629:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.511635:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.511670:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.511673:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80271 00000080:00010000:0.0:1769074479.511685:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80271 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.511689:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.511697:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.511706:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.511911:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-49, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.511953:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.512009:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad400/0x68a7113811d7f948 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ea08 expref: -99 pid: 30783 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.512022:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x5b:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.512069:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad400/0x68a7113811d7f948 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ea08 expref: -99 pid: 30783 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.512078:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.512084:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.512087:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad400/0x68a7113811d7f948 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ea08 expref: -99 pid: 30783 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.512110:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80271 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f29 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.512134:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.512150:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005663480 x1855007612333568/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.512188:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333568:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.512606:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.520170:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333568:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.520567:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x5b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.520588:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.520627:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x5b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.520730:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x5b:0x0](ffff8b10181900c8) 00000008:00010000:0.0:1769074479.520772:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074467 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.520878:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.520901:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.521093:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.521103:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.521115:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.521143:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333696:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.521246:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.527831:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333696:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.527867:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527889:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527902:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527915:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527928:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527936:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.527944:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.528081:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660a80 x1855007612333696/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.528101:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.528118:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.528127:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.528163:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.528170:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80278 00000080:00010000:0.0:1769074479.528177:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.528181:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80278 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.528185:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.528196:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.528229:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.528532:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-5, flags=8 00000080:00200008:0.0:1769074479.528549:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-5:[0x200001b71:0x2f:0x0](ffff8b1018123a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.528556:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x2f:0x0](ffff8b1018123a88),name=f5.replay-single-5 00000002:00010000:0.0:1769074479.528579:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x2f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.528591:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.528604:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.528613:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.528618:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x2f:0x0](ffff8b1018123a88) 00000080:00010000:0.0:1769074479.528621:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x2f:0x0](ffff8b1018123a88) for lock 0x68a7113811d7f7dc 00000080:00010000:0.0:1769074479.528626:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7dc from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.528629:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.528636:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.528715:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x2f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.528731:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x2f:0x0] 00000008:00010000:0.0:1769074479.528759:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.528764:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.528788:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.528798:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.528812:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.528816:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.528824:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.528835:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.528849:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x2f:0x0] updating i_size 6 i_blocks 1 00000020:00010000:0.0:1769074479.528854:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.528857:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.528861:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.528865:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.528867:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.528879:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x2f:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.528918:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.528925:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.528932:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.528944:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.528950:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.528955:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.529146:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80278 00000080:00010000:0.0:1769074479.529153:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80278 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.529156:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.529165:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.529174:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.529184:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-5, flags=0 00000080:00200000:0.0:1769074479.529189:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.529197:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.529203:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.529210:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.529216:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.529220:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.529222:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80278 00000080:00010000:0.0:1769074479.529226:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80278 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.529229:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.529527:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.529537:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.529544:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-5, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.529575:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.529615:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.529627:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x2f:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.529891:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.529901:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.529908:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.529911:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602cc00/0x68a7113811d7f7dc lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x2f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e324 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.529931:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80278 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f53 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.529951:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.530149:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005660a80 x1855007612333952/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.530188:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333952:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.530630:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.537511:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612333952:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.537605:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x2f:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.537624:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.537701:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x2f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.537769:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x2f:0x0](ffff8b1018123a88) 00000008:00010000:0.0:1769074479.537815:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 6 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 6 blocks 1 [cma]time 1769074407 1769074407 1769074465 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.537920:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.537944:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.538505:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.538519:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.538531:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.538562:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334080:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.538819:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.543867:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334080:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.543907:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.543931:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.543946:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544118:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544136:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544145:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544154:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.544168:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005663100 x1855007612334080/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.544184:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544200:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.544208:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.544255:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.544408:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8027f 00000080:00010000:0.0:1769074479.544419:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.544425:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8027f from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.544429:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544440:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.544472:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.544488:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-50, flags=8 00000080:00200008:0.0:1769074479.544504:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-50:[0x200001b71:0x5c:0x0](ffff8b1004812a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.544510:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x5c:0x0](ffff8b1004812a08),name=f5.replay-single-50 00000002:00010000:0.0:1769074479.544535:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x5c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.544547:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.544560:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.544567:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.544572:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x5c:0x0](ffff8b1004812a08) 00000080:00010000:0.0:1769074479.544576:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x5c:0x0](ffff8b1004812a08) for lock 0x68a7113811d7f94f 00000080:00010000:0.0:1769074479.544580:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f94f from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.544584:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.544591:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.544616:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x5c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.544630:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x5c:0x0] 00000008:00010000:0.0:1769074479.544698:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.544703:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.544724:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.544736:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.544750:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.544754:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.544763:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.544774:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.544789:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x5c:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.544794:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.544797:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.544801:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.544806:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.544809:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.544821:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x5c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.544860:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.544868:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.544875:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544889:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.544896:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.544901:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.544904:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8027f 00000080:00010000:0.0:1769074479.544908:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8027f from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.544911:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.544919:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.544928:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.544936:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-50, flags=0 00000080:00200000:0.0:1769074479.544941:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.544947:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.544952:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.545130:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.545139:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.545144:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.545147:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8027f 00000080:00010000:0.0:1769074479.545151:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8027f from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.545154:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.545164:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.545173:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.545179:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-50, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.545208:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.545469:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.545484:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x5c:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.545527:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.545535:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.545541:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.545545:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac200/0x68a7113811d7f94f lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x5c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ea2b expref: -99 pid: 30784 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.545570:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d8027f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3061f7d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.545594:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.545607:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005663100 x1855007612334336/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.545901:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334336:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.546186:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.553571:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334336:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.553767:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x5c:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.553786:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.553823:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x5c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.553898:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x5c:0x0](ffff8b1004812a08) 00000008:00010000:0.0:1769074479.553940:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074467 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.554537:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.554568:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.554700:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.554710:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.554721:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.554750:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334464:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.554928:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.559526:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612334464:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.559560:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.559579:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.559595:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.559608:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.559622:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.559631:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.562060:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.562102:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660e00 x1855007612334464/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.562123:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80286 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3061fa7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.562139:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.562147:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.618561:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-54, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.618608:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d8029b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062025 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.618701:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac400/0x68a7113811d7f96b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eab7 expref: -99 pid: 30788 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.618715:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x60:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.618769:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac400/0x68a7113811d7f96b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eab7 expref: -99 pid: 30788 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.618777:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d8029b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062025 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.618783:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.618785:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cac400/0x68a7113811d7f96b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eab7 expref: -99 pid: 30788 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.618807:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d8029b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062025 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.618825:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.618840:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005662a00 x1855007612335872/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.618869:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612335872:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.619460:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00010000:00010000:0.0:1769074479.650828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074479.668944:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-55:[0x200001b71:0x61:0x0](ffff8b101817ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.668957:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x61:0x0](ffff8b101817ec08),name=f5.replay-single-55 00000002:00010000:0.0:1769074479.669113:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x61:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.669128:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.669143:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.669150:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.669158:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x61:0x0](ffff8b101817ec08) 00000080:00010000:0.0:1769074479.669161:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x61:0x0](ffff8b101817ec08) for lock 0x68a7113811d7f972 00000080:00010000:0.0:1769074479.669167:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f972 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.669406:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.669419:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.669449:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x61:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.669465:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x61:0x0] 00000008:00010000:0.0:1769074479.669492:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.669496:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.669519:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.669530:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.669544:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.669549:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.669556:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.669568:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.669583:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x61:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.669588:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.669591:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.669595:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.669599:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.669603:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.669616:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x61:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.669900:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.669913:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.669922:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.669933:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.669940:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.669946:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.669949:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802a2 00000080:00010000:0.0:1769074479.669953:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802a2 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.669957:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.670414:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.670427:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.670441:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-55, flags=0 00000080:00200000:0.0:1769074479.670448:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.670458:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.670464:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.670472:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.670479:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.670484:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.670487:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802a2 00000080:00010000:0.0:1769074479.670490:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802a2 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.670493:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.670500:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.670507:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.670513:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-55, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.670548:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.670588:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.670601:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x61:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.670770:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.670783:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.670788:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.670791:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005cad800/0x68a7113811d7f972 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eada expref: -99 pid: 30789 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.670815:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d802a2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306204f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.670837:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.670852:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005663800 x1855007612336256/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.670883:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005663800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336256:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.671188:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.687118:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005663800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336256:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.687215:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x61:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.687247:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.687290:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x61:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.687380:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x61:0x0](ffff8b101817ec08) 00000008:00010000:0.0:1769074479.687425:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.687535:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.687562:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.687954:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.688085:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.688099:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.688130:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336384:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.693747:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.697293:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336384:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.697340:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697356:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697371:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697385:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697400:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697407:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697414:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.697426:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660700 x1855007612336384/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.697442:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697457:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.697465:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.697508:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.697516:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802a9 00000080:00010000:0.0:1769074479.697524:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.697528:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802a9 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.697532:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.697542:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.697576:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.697593:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-56, flags=8 00000080:00200008:0.0:1769074479.697615:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-56:[0x200001b71:0x62:0x0](ffff8b1004816c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.697622:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x62:0x0](ffff8b1004816c08),name=f5.replay-single-56 00000002:00010000:0.0:1769074479.697914:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x62:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.697928:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.697942:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.697950:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.697955:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x62:0x0](ffff8b1004816c08) 00000080:00010000:0.0:1769074479.698079:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x62:0x0](ffff8b1004816c08) for lock 0x68a7113811d7f979 00000080:00010000:0.0:1769074479.698085:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f979 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.698089:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.698097:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.698128:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x62:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.698144:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x62:0x0] 00000008:00010000:0.0:1769074479.698173:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.698178:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972db30, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.698202:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.698213:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.698227:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.698364:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.698375:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.698393:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.698410:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x62:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.698415:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.698419:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.698422:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972db30 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.698427:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.698430:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972db30 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.698444:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x62:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.698487:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.698497:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.698505:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.698514:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.698520:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.698525:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.698528:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802a9 00000080:00010000:0.0:1769074479.698532:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802a9 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.698535:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.698543:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.698553:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.698563:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-56, flags=0 00000080:00200000:0.0:1769074479.698569:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.698576:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.698581:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.698590:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.698597:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.698601:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.698604:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802a9 00000080:00010000:0.0:1769074479.698608:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802a9 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.698612:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.698620:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.698628:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.698634:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-56, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.698870:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.698918:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.698931:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x62:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.699145:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.699162:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.699169:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.699173:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5600/0x68a7113811d7f979 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eafd expref: -99 pid: 30790 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.699198:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802a9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062079 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.699222:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.699247:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005660700 x1855007612336640/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.699282:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336640:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.699422:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.719849:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336640:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.719941:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x62:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.719973:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.720007:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x62:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.720079:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x62:0x0](ffff8b1004816c08) 00000008:00010000:0.0:1769074479.720120:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074479.720224:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.720263:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.720353:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.720362:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074479.720375:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074479.720401:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336768:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074479.720480:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.731898:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612336768:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.731935:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.731952:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.731978:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.731992:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732006:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732015:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732022:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.732037:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005662680 x1855007612336768/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.732054:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732068:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.732076:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.732113:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.732121:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b0 00000080:00010000:0.0:1769074479.732128:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.732131:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b0 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.732135:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732143:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.732171:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.732185:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-57, flags=8 00000080:00200008:0.0:1769074479.732200:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-57:[0x200001b71:0x63:0x0](ffff8b10181721c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.732206:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x63:0x0](ffff8b10181721c8),name=f5.replay-single-57 00000002:00010000:0.0:1769074479.732230:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x63:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.732251:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.732263:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.732271:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.732276:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x63:0x0](ffff8b10181721c8) 00000080:00010000:0.0:1769074479.732279:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x63:0x0](ffff8b10181721c8) for lock 0x68a7113811d7f980 00000080:00010000:0.0:1769074479.732283:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f980 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.732286:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.732292:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.732682:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x63:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.732699:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x63:0x0] 00000008:00010000:0.0:1769074479.732724:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.732729:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972db30, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.732751:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.732761:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.732774:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.732778:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.732786:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.732803:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.732819:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x63:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.732824:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.732827:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.732831:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972db30 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.732835:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.732838:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972db30 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.732849:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x63:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.732888:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.732897:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.732909:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732918:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.732925:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.732929:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.732932:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b0 00000080:00010000:0.0:1769074479.732937:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b0 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.732940:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.732946:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.732955:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.733196:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-57, flags=0 00000080:00200000:0.0:1769074479.733202:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.733210:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.733217:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.733227:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.733410:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.733416:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.733419:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b0 00000080:00010000:0.0:1769074479.733424:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b0 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.733427:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.733436:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.733446:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.733453:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-57, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.733484:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.733524:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.733537:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x63:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.733580:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.733594:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.733600:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.733603:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4000/0x68a7113811d7f980 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x63:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb20 expref: -99 pid: 30791 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.733626:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d802b0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620a3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.733771:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.733788:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005662680 x1855007612337024/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.733819:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337024:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.733912:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.740797:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337024:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.740891:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x63:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.740910:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.740947:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x63:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.741157:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x63:0x0](ffff8b10181721c8) 00000008:00010000:0.0:1769074479.741198:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.747198:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.747855:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.747941:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.747953:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.748423:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.748482:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337152:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.748688:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.757636:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337152:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074479.757711:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757726:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757740:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757754:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757767:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757775:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757783:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074479.757794:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006042d80 x1855007612337152/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074479.757810:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757825:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074479.757833:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.757871:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.757879:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b7 00000080:00010000:0.0:1769074479.757886:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.757890:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b7 from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.757893:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.757902:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.757931:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.757946:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-58, flags=8 00000080:00200008:0.0:1769074479.757961:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-58:[0x200001b71:0x64:0x0](ffff8b10181700c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.757967:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x64:0x0](ffff8b10181700c8),name=f5.replay-single-58 00000002:00010000:0.0:1769074479.757993:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x64:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.758004:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.758015:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.758021:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.758027:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x64:0x0](ffff8b10181700c8) 00000080:00010000:0.0:1769074479.758030:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x64:0x0](ffff8b10181700c8) for lock 0x68a7113811d7f987 00000080:00010000:0.0:1769074479.758034:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f987 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.758037:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.758045:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.758070:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x64:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.758085:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x64:0x0] 00000008:00010000:0.0:1769074479.758112:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.758117:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.758140:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.758151:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.758166:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.758171:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.758178:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.758189:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.758205:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x64:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.758209:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.758213:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.758216:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.758221:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.758224:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.758237:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x64:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.758281:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.758291:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.758298:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758312:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.758319:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.758323:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.758400:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b7 00000080:00010000:0.0:1769074479.758405:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b7 from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.758409:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758415:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.758424:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.758433:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-58, flags=0 00000080:00200000:0.0:1769074479.758438:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.758445:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.758451:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758458:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.758465:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.758469:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.758471:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802b7 00000080:00010000:0.0:1769074479.758474:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802b7 from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.758477:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758485:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.758494:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.758501:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-58, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.758530:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758570:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.758583:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x64:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.758627:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.758636:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.758699:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.758702:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69400/0x68a7113811d7f987 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x64:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb43 expref: -99 pid: 30792 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.758726:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d802b7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620cd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.758748:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.758760:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005661c00 x1855007612337408/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.758798:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337408:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.758905:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074479.767322:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337408:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074479.768900:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x64:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074479.768921:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074479.773814:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x64:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.774923:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x64:0x0](ffff8b10181700c8) 00000008:00010000:1.0:1769074479.774998:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.775128:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.775158:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.775249:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.775266:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.775276:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.775320:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337536:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.775446:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000080:00200000:3.0:1769074479.791619:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.791703:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.791721:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.791736:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.791744:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.791754:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.791758:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802be 00000080:00010000:3.0:1769074479.791766:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802be from it ffffaa914211fc10 00010000:00010000:3.0:1769074479.791770:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.791778:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.791794:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.791808:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-59, flags=0 00000080:00200000:3.0:1769074479.791815:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.791821:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.791827:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.791835:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.791842:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.791846:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.791849:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802be 00000080:00010000:3.0:1769074479.791852:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802be from it ffffaa914211fd68 00010000:00010000:3.0:1769074479.791856:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.791863:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.791870:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074479.791876:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-59, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074479.791913:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.791956:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68c00/0x68a7113811d7f98e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eb66 expref: -99 pid: 30793 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.791969:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x65:0x0]: losing layout lock 00010000:00010000:3.0:1769074479.792016:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68c00/0x68a7113811d7f98e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb66 expref: -99 pid: 30793 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.792024:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.792031:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074479.792034:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68c00/0x68a7113811d7f98e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eb66 expref: -99 pid: 30793 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.792055:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d802be lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30620f7 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074479.792076:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074479.792091:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037ddf80 x1855007612337792/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074479.792123:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337792:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074479.792238:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.814758:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337792:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074479.814852:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x65:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074479.814871:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074479.814905:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x65:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.814972:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x65:0x0](ffff8b1018170908) 00000008:00010000:3.0:1769074479.815014:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074479.815122:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.815144:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.815213:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.815225:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.815250:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074479.817618:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337920:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074479.817851:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.836247:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612337920:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074479.836287:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836310:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836325:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836339:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836359:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836369:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836377:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074479.836390:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dfb80 x1855007612337920/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074479.836406:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836422:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074479.836430:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074479.836471:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074479.836478:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802c5 00000080:00010000:3.0:1769074479.836484:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.836488:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802c5 from it ffffaa914211fb58 00010000:00010000:3.0:1769074479.836492:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836502:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.836532:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.836547:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-6, flags=8 00000080:00200008:3.0:1769074479.836563:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-6:[0x200001b71:0x30:0x0](ffff8b101817a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074479.836570:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x30:0x0](ffff8b101817a1c8),name=f5.replay-single-6 00000002:00010000:3.0:1769074479.836593:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x30:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.836603:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.836616:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.836624:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.836628:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x30:0x0](ffff8b101817a1c8) 00000080:00010000:3.0:1769074479.836631:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x30:0x0](ffff8b101817a1c8) for lock 0x68a7113811d7f7e3 00000080:00010000:3.0:1769074479.836635:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7e3 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074479.836667:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.836685:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.836711:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x30:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074479.836727:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x30:0x0] 00000008:00010000:3.0:1769074479.836754:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074479.836758:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76530, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074479.836781:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.836792:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.836810:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074479.836814:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074479.836823:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074479.836836:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074479.836851:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x30:0x0] updating i_size 6 i_blocks 1 00000020:00010000:3.0:1769074479.836856:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:3.0:1769074479.836860:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.836863:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76530 (ffff8b10056851b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074479.836867:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074479.836870:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76530 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074479.836882:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x30:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.836921:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.836930:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.836936:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836945:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.836951:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.836955:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.836957:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802c5 00000080:00010000:3.0:1769074479.836976:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802c5 from it ffffaa914211fc10 00010000:00010000:3.0:1769074479.836979:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.836986:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.836994:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074479.837002:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-6, flags=0 00000080:00200000:3.0:1769074479.837007:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.837014:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.837019:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.837027:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.837033:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074479.837037:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074479.837039:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802c5 00000080:00010000:3.0:1769074479.837043:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802c5 from it ffffaa914211fd68 00010000:00010000:3.0:1769074479.837045:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.837052:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074479.837060:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074479.837066:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-6, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074479.837094:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.837133:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074479.837145:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x30:0x0]: losing layout lock 00010000:00010000:3.0:1769074479.837185:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.837194:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.837201:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074479.837204:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100602d400/0x68a7113811d7f7e3 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e340 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074479.837226:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802c5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062121 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074479.837248:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074479.837259:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dfb80 x1855007612338176/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074479.837288:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338176:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074479.837385:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.843450:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338176:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074479.843539:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x30:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074479.843559:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074479.843597:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074479.843710:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x30:0x0](ffff8b101817a1c8) 00000008:00010000:3.0:1769074479.843756:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 6 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 6 blocks 1 [cma]time 1769074407 1769074407 1769074465 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074479.843865:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074479.843888:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074479.844163:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.844174:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074479.844188:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074479.844218:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338304:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074479.844348:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074479.851484:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338304:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074479.851519:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851533:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851546:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851558:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851574:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851583:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851591:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074479.851604:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc380 x1855007612338304/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074479.851619:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169400/0x68a7113811d802cc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306214b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074479.851634:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074479.851882:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:1.0:1769074479.884772:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005690000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338688:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.884808:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884822:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884835:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884873:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884888:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884896:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884903:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.884915:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005690000 x1855007612338688/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.884934:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.884951:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.884959:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074479.884997:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074479.885005:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802d3 00000080:00010000:1.0:1769074479.885011:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.885014:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802d3 from it ffffaa914211fb58 00010000:00010000:1.0:1769074479.885018:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.885026:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.885063:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.885078:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-61, flags=8 00000080:00200008:1.0:1769074479.885094:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-61:[0x200001b71:0x67:0x0](ffff8b1004810908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074479.885101:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x67:0x0](ffff8b1004810908),name=f5.replay-single-61 00000002:00010000:1.0:1769074479.885128:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x67:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.885142:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.885152:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.885158:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.885162:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x67:0x0](ffff8b1004810908) 00000080:00010000:1.0:1769074479.885165:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x67:0x0](ffff8b1004810908) for lock 0x68a7113811d7f99c 00000080:00010000:1.0:1769074479.885169:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f99c from it ffffaa914211fcb8 00010000:00010000:1.0:1769074479.885171:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.885177:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.885200:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x67:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074479.885213:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x67:0x0] 00000008:00010000:1.0:1769074479.885252:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074479.885256:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f26b0, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074479.885277:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.885564:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.885580:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074479.885584:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074479.885591:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074479.885602:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074479.885619:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x67:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074479.885624:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074479.885627:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.885630:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f26b0 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074479.885634:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074479.885688:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f26b0 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074479.885702:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x67:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.885742:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.885752:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.885759:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.885768:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.885774:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.885785:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.885788:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802d3 00000080:00010000:1.0:1769074479.885793:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802d3 from it ffffaa914211fc10 00010000:00010000:1.0:1769074479.885796:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.885803:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.886428:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074479.886443:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-61, flags=0 00000080:00200000:1.0:1769074479.886449:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.886459:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.886467:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.886478:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.886484:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074479.886490:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074479.886492:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802d3 00000080:00010000:1.0:1769074479.886496:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802d3 from it ffffaa914211fd68 00010000:00010000:1.0:1769074479.886498:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.886504:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074479.886512:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074479.886527:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-61, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074479.886559:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.886598:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074479.886609:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x67:0x0]: losing layout lock 00010000:00010000:1.0:1769074479.886691:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.886699:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.886705:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074479.886708:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655a00/0x68a7113811d7f99c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ebac expref: -99 pid: 30795 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074479.886732:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d802d3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062175 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074479.886753:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074479.886767:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041880 x1855007612338944/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074479.886795:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338944:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074479.886959:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.899471:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612338944:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074479.899560:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x67:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074479.899578:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074479.905083:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x67:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074479.905179:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x67:0x0](ffff8b1004810908) 00000008:00010000:1.0:1769074479.905378:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074408 1769074408 0 by oinfo size 7 blocks 1 [cma]time 1769074408 1769074408 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074479.905509:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074479.905536:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074479.905623:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.905631:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074479.905706:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074479.905740:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612339072:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074479.905858:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074479.911489:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612339072:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074479.911527:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911543:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911558:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911571:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911612:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911621:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911629:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074479.911681:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006042a00 x1855007612339072/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074495 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074479.911700:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074479.911715:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074479.911723:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074479.911830:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074479.911851:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802da 00000080:00010000:0.0:1769074479.911861:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.911865:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802da from it ffffaa914211fb58 00010000:00010000:0.0:1769074479.911871:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.911886:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.911923:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.911936:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-62, flags=8 00000080:00200008:0.0:1769074479.911948:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-62:[0x200001b71:0x68:0x0](ffff8b10048142c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074479.911953:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x68:0x0](ffff8b10048142c8),name=f5.replay-single-62 00000002:00010000:0.0:1769074479.911973:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x68:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.911986:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.911997:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.912008:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.912031:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x68:0x0](ffff8b10048142c8) 00000080:00010000:0.0:1769074479.912035:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x68:0x0](ffff8b10048142c8) for lock 0x68a7113811d7f9a3 00000080:00010000:0.0:1769074479.912039:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9a3 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074479.912042:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.912057:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.912078:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x68:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074479.912091:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x68:0x0] 00000008:00010000:0.0:1769074479.912111:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074479.912115:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074479.912131:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.912141:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.912153:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074479.912157:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074479.912165:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074479.912180:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074479.912193:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x68:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074479.912197:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074479.912201:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.912204:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074479.912208:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074479.912211:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074479.912222:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x68:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074479.912258:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.912266:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.912272:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912280:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.912285:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.912289:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.912292:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802da 00000080:00010000:0.0:1769074479.912308:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802da from it ffffaa914211fc10 00010000:00010000:0.0:1769074479.912312:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912319:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.912328:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074479.912336:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-62, flags=0 00000080:00200000:0.0:1769074479.912340:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074479.912346:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074479.912351:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912359:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.912365:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074479.912369:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074479.912372:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802da 00000080:00010000:0.0:1769074479.912375:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802da from it ffffaa914211fd68 00010000:00010000:0.0:1769074479.912377:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912384:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074479.912392:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074479.912398:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-62, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074479.912431:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912462:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074479.912472:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x68:0x0]: losing layout lock 00010000:00010000:0.0:1769074479.912514:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.912526:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074479.912532:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074479.912535:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654800/0x68a7113811d7f9a3 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x68:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ebcf expref: -99 pid: 30797 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074479.912554:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9800/0x68a7113811d802da lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306219f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074479.912572:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074479.912586:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005661500 x1855007612339328/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074479.912611:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612339328:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074479.912757:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2780 going to sleep for 11 seconds 00010000:00010000:3.0:1769074480.017188:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.017221:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.017227:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.017237:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.017255:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.017276:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x6b:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074480.017281:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.017284:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.017289:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034efeb0 (ffff8b1007c696c8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.017363:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.017367:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034efeb0 (ffff8b1005685128/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.017385:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.017495:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.017517:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.017531:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017542:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.017549:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.017556:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.017560:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802ef 00000080:00010000:3.0:1769074480.017566:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802ef from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.017568:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017575:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.017586:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.017596:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-65, flags=0 00000080:00200000:3.0:1769074480.017602:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.017608:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.017614:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017621:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.017628:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.017631:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.017633:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802ef 00000080:00010000:3.0:1769074480.017636:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802ef from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.017665:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017684:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.017692:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.017698:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-65, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.017725:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017759:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69800/0x68a7113811d7f9b8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ec38 expref: -99 pid: 30800 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.017769:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x6b:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.017817:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69800/0x68a7113811d7f9b8 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x6b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec38 expref: -99 pid: 30800 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.017825:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.017835:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.017838:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69800/0x68a7113811d7f9b8 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x6b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec38 expref: -99 pid: 30800 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.017860:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d802ef lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306221d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.017877:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.017892:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc000 x1855007612340480/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.017917:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340480:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.018163:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.039612:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340480:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.039726:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x6b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.039742:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.039773:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.039912:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x6b:0x0](ffff8b1004817448) 00000008:00010000:3.0:1769074480.039945:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.040033:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.040696:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.040774:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.040785:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.040796:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.041763:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340608:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.042403:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.047889:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340608:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.047916:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047928:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047940:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047950:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047962:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047970:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.047977:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.047987:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dd880 x1855007612340608/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.048001:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.048014:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.048219:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.048245:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.048251:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802f6 00000080:00010000:0.0:1769074480.048257:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.048260:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802f6 from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.048264:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.048272:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.048733:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.048748:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-66, flags=8 00000080:00200008:0.0:1769074480.048761:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-66:[0x200001b71:0x6c:0x0](ffff8b10181921c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.048767:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x6c:0x0](ffff8b10181921c8),name=f5.replay-single-66 00000002:00010000:0.0:1769074480.048787:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x6c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.048798:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.048809:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.048826:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.048830:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x6c:0x0](ffff8b10181921c8) 00000080:00010000:0.0:1769074480.048833:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x6c:0x0](ffff8b10181921c8) for lock 0x68a7113811d7f9bf 00000080:00010000:0.0:1769074480.048837:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9bf from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.048841:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.048848:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.048868:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x6c:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:0.0:1769074480.048879:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x6c:0x0] 00000008:00010000:0.0:1769074480.048897:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.048900:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.048914:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.048923:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.048940:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.048943:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.048951:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.048960:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.048971:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x6c:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.048976:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.048978:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.048981:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.048985:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.048988:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.048997:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.049037:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.049044:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.051512:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051540:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.051551:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.051564:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.051568:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802f6 00000080:00010000:0.0:1769074480.051580:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802f6 from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.051584:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051592:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.051610:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.051625:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-66, flags=0 00000080:00200000:0.0:1769074480.051631:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.051681:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.051691:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051698:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.051705:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.051709:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.051712:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802f6 00000080:00010000:0.0:1769074480.051715:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802f6 from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.051718:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051725:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.051733:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.051738:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-66, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.051777:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051812:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.051822:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x6c:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.051866:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.051874:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.051879:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.051881:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c68a00/0x68a7113811d7f9bf lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x6c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec5b expref: -99 pid: 30801 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.051900:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168400/0x68a7113811d802f6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062247 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.051919:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.051931:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005662300 x1855007612340864/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.051955:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005662300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340864:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.053384:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.063792:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005662300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340864:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.063876:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x6c:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074480.063892:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.063919:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.063968:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x6c:0x0](ffff8b10181921c8) 00000008:00010000:0.0:1769074480.063999:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074468 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.064106:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.064125:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.064184:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.064193:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.064202:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.064222:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340992:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.064292:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.073537:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612340992:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.073562:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073576:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073587:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073597:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073612:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073619:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073626:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.073636:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005661f80 x1855007612340992/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.073698:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073712:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.073719:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.073748:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.073754:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802fd 00000080:00010000:0.0:1769074480.073759:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.073763:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802fd from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.073766:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.073773:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.073798:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.073810:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-67, flags=8 00000080:00200008:0.0:1769074480.073835:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-67:[0x200001b71:0x6d:0x0](ffff8b101817c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.073841:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x6d:0x0](ffff8b101817c2c8),name=f5.replay-single-67 00000002:00010000:0.0:1769074480.073861:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x6d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.073871:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.073882:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.073889:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.073893:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x6d:0x0](ffff8b101817c2c8) 00000080:00010000:0.0:1769074480.073895:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x6d:0x0](ffff8b101817c2c8) for lock 0x68a7113811d7f9c6 00000080:00010000:0.0:1769074480.073899:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9c6 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.073901:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.073908:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.073930:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x6d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074480.073943:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x6d:0x0] 00000008:00010000:0.0:1769074480.073963:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.073966:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972db30, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.073985:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.073994:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.074010:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.074014:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.074021:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.074031:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.074043:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x6d:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.074059:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.074063:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.074066:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972db30 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.074070:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.074072:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972db30 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.074082:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6d:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.074118:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.074126:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.074132:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074139:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.074145:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.074149:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.074151:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802fd 00000080:00010000:0.0:1769074480.074156:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802fd from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.074158:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074164:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.074172:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.074179:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-67, flags=0 00000080:00200000:0.0:1769074480.074183:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.074188:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.074193:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074200:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.074205:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.074208:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.074210:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d802fd 00000080:00010000:0.0:1769074480.074213:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d802fd from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.074216:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074221:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.074228:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.074233:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-67, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.074254:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074281:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.074291:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x6d:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.074423:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.074430:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.074435:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.074438:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655800/0x68a7113811d7f9c6 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x6d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ec7e expref: -99 pid: 30802 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.074455:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d802fd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062271 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.074473:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.074482:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005661f80 x1855007612341248/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.074504:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612341248:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.074570:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a26c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.087509:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005661f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612341248:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.087581:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x6d:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074480.087598:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.087626:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x6d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.087744:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x6d:0x0](ffff8b101817c2c8) 00000008:00010000:0.0:1769074480.087780:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074468 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.087869:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.087886:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.087941:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80304 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.087950:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80304 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.087959:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80304 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.087978:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612341376:192.168.204.102@tcp:101:rm.0 00000080:00200000:3.0:1769074480.160109:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.160147:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.160441:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.160453:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.160467:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.160501:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342528:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.160633:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.165944:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342528:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.166402:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166420:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166436:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166451:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166471:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166480:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166489:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.166503:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de680 x1855007612342528/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.166521:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166537:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.166545:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.166590:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.166598:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80319 00000080:00010000:3.0:1769074480.166605:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.166609:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80319 from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.166612:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.166621:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.166697:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.166716:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-70, flags=8 00000080:00200008:3.0:1769074480.166733:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-70:[0x200001b71:0x70:0x0](ffff8b1004819148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.166740:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x70:0x0](ffff8b1004819148),name=f5.replay-single-70 00000002:00010000:3.0:1769074480.166765:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x70:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.166777:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.166789:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.166796:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.166801:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x70:0x0](ffff8b1004819148) 00000080:00010000:3.0:1769074480.166816:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x70:0x0](ffff8b1004819148) for lock 0x68a7113811d7f9db 00000080:00010000:3.0:1769074480.166821:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9db from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.166825:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.166833:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.167478:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x70:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.167501:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x70:0x0] 00000008:00010000:3.0:1769074480.167533:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.167539:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76830, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074480.167561:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.167574:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.167593:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.167597:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.167606:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.167618:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.167634:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x70:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074480.167678:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.167682:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.167686:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76830 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.167691:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.167694:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76830 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.167708:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x70:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.167752:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.167765:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.167775:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.167784:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.167792:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.167797:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.167799:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80319 00000080:00010000:3.0:1769074480.167818:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80319 from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.167821:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.167829:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.167839:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.167850:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-70, flags=0 00000080:00200000:3.0:1769074480.167855:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.167862:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.169271:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.169290:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.169300:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.169308:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.169311:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80319 00000080:00010000:3.0:1769074480.169320:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80319 from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.169324:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.169332:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.169348:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.169357:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-70, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.169405:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.169445:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.169458:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x70:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.169512:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.169520:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.169527:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.169530:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c4a00/0x68a7113811d7f9db lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x70:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ece7 expref: -99 pid: 30805 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.169552:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80319 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062319 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.169575:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.169592:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de680 x1855007612342784/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.169626:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342784:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.169791:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.182371:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342784:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.182465:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x70:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.182486:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.182527:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x70:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.182602:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x70:0x0](ffff8b1004819148) 00000008:00010000:3.0:1769074480.184055:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.184204:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.184385:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.184477:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.184487:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.184498:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.184531:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342912:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.184691:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.197912:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612342912:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.197951:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198103:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198119:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198132:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198147:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198155:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198162:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.198175:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005660e00 x1855007612342912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.198192:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.198208:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.198216:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.199147:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.199166:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80320 00000080:00010000:0.0:1769074480.199175:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.199180:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80320 from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.199185:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.199199:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.199400:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.199417:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-71, flags=8 00000080:00200008:0.0:1769074480.199435:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-71:[0x200001b71:0x71:0x0](ffff8b1004818908), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.199441:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x71:0x0](ffff8b1004818908),name=f5.replay-single-71 00000002:00010000:0.0:1769074480.199470:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x71:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.199485:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.199498:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.199506:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.199511:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x71:0x0](ffff8b1004818908) 00000080:00010000:0.0:1769074480.199515:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x71:0x0](ffff8b1004818908) for lock 0x68a7113811d7f9e2 00000080:00010000:0.0:1769074480.199520:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9e2 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.199523:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.199530:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.199556:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x71:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074480.199570:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x71:0x0] 00000008:00010000:0.0:1769074480.199600:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.199604:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.199626:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.199637:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.205475:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.205487:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.205504:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.205530:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.205560:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x71:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.205567:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.205571:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.205578:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.205584:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.205587:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.205612:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x71:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.205702:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.205732:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.205753:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.205775:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.205784:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.205792:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.205797:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80320 00000080:00010000:0.0:1769074480.205803:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80320 from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.205807:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.205815:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.205828:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.205843:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-71, flags=0 00000080:00200000:0.0:1769074480.205850:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.205857:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.205863:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.205872:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.205878:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.205883:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.205886:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80320 00000080:00010000:0.0:1769074480.205890:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80320 from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.205893:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.205901:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.205909:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.205916:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-71, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.205956:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.206000:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.206013:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x71:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.206062:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.206071:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.206077:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.206080:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c69000/0x68a7113811d7f9e2 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed0a expref: -99 pid: 30806 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.206105:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80320 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062343 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.206127:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.206143:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005660e00 x1855007612343168/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.206181:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343168:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.206317:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.215621:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005660e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343168:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.215795:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x71:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074480.215816:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.215856:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x71:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.215934:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x71:0x0](ffff8b1004818908) 00000008:00010000:0.0:1769074480.215991:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.216103:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.216128:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.216209:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.216218:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.216231:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.216256:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343296:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.216371:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.222465:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343296:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.222504:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222527:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222542:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222556:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222571:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222580:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222588:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.222601:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c0380 x1855007612343296/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.222622:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222686:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.222695:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.222734:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.222742:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80327 00000080:00010000:0.0:1769074480.222750:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.222755:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80327 from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.222759:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.222770:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.223069:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.223088:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-72, flags=8 00000080:00200008:0.0:1769074480.223106:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-72:[0x200001b71:0x72:0x0](ffff8b100481c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.223113:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x72:0x0](ffff8b100481c2c8),name=f5.replay-single-72 00000002:00010000:0.0:1769074480.223142:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x72:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.223155:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.223169:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.223176:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.223181:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x72:0x0](ffff8b100481c2c8) 00000080:00010000:0.0:1769074480.223184:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x72:0x0](ffff8b100481c2c8) for lock 0x68a7113811d7f9e9 00000080:00010000:0.0:1769074480.223189:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9e9 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.223192:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.223200:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.223233:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x72:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074480.223251:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x72:0x0] 00000008:00010000:0.0:1769074480.223284:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.223288:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d5b0, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.223312:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.223323:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.223337:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.223341:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.223348:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.223361:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.223379:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x72:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.223384:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.223387:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.223391:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d5b0 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.223395:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80ff8 (ffff8b1007c693f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.223398:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d5b0 (ffff8b1007c693f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.223413:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x72:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.223464:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.223473:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.223479:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.223492:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.223498:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.223502:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.223505:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80327 00000080:00010000:0.0:1769074480.223509:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80327 from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.223512:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.223519:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.223528:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.223536:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-72, flags=0 00000080:00200000:0.0:1769074480.223541:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.223548:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.223553:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.223560:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.223566:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.223570:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.223573:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80327 00000080:00010000:0.0:1769074480.223576:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80327 from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.223579:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.223587:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.223595:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.223601:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-72, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.223633:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.224431:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.224448:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x72:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.225399:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.225414:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.225421:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.225424:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e200/0x68a7113811d7f9e9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed2d expref: -99 pid: 30807 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.225450:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80327 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306236d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.225505:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.225522:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006041500 x1855007612343552/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.225554:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343552:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.226135:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.234388:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006041500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343552:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.234483:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x72:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074480.234503:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.235536:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x72:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.235619:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x72:0x0](ffff8b100481c2c8) 00000008:00010000:0.0:1769074480.236494:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.236633:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.236706:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.236813:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.236823:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.236839:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.236878:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343680:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.239222:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.245435:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343680:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.245473:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245488:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245502:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245516:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245530:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245538:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245547:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.245560:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c0380 x1855007612343680/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.245576:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245592:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.245600:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.245880:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.245892:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8032e 00000080:00010000:0.0:1769074480.245900:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.245904:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8032e from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.245907:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.245917:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.245950:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.246038:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-73, flags=8 00000080:00200008:0.0:1769074480.246057:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-73:[0x200001b71:0x73:0x0](ffff8b100481e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.246064:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x73:0x0](ffff8b100481e3c8),name=f5.replay-single-73 00000002:00010000:0.0:1769074480.246096:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x73:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.246109:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.246121:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.246128:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.246132:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x73:0x0](ffff8b100481e3c8) 00000080:00010000:0.0:1769074480.246136:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x73:0x0](ffff8b100481e3c8) for lock 0x68a7113811d7f9f0 00000080:00010000:0.0:1769074480.246140:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9f0 from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.246143:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.246151:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.246178:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x73:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074480.246194:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x73:0x0] 00000008:00010000:0.0:1769074480.246222:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.246391:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d130, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.246415:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.246426:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.246442:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.246447:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.246454:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.246467:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.246484:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x73:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.246489:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.246492:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.246495:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d130 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.246499:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.246502:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d130 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.246514:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x73:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.246559:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.246569:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.246576:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.246584:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.246591:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.246595:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.246598:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8032e 00000080:00010000:0.0:1769074480.246601:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8032e from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.246605:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.246611:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.246619:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.246629:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-73, flags=0 00000080:00200000:0.0:1769074480.246634:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.247175:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.247186:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.247201:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.247207:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.247212:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.247214:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8032e 00000080:00010000:0.0:1769074480.247218:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8032e from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.247221:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.247402:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.247416:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.247423:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-73, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.247461:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.247497:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.247509:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x73:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.247561:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.247569:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.247575:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.247578:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771600/0x68a7113811d7f9f0 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x73:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed50 expref: -99 pid: 30808 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.247599:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d8032e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062397 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.247621:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.247635:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c0380 x1855007612343936/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.248096:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612343936:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.248266:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecd80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.268534:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344064:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.268572:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268588:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268601:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268615:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268630:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268654:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268662:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.268690:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c46300 x1855007612344064/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.268707:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268726:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.268734:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.268777:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.268869:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80335 00000080:00010000:3.0:1769074480.268878:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.268882:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80335 from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.268885:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.268895:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.268930:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.268946:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-74, flags=8 00000080:00200008:3.0:1769074480.268962:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-74:[0x200001b71:0x74:0x0](ffff8b1004819988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.268969:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x74:0x0](ffff8b1004819988),name=f5.replay-single-74 00000002:00010000:3.0:1769074480.268999:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x74:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.269012:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.269025:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.269033:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.269037:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x74:0x0](ffff8b1004819988) 00000080:00010000:3.0:1769074480.269041:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x74:0x0](ffff8b1004819988) for lock 0x68a7113811d7f9f7 00000080:00010000:3.0:1769074480.269046:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9f7 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.269049:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.269057:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.269085:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x74:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.269100:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x74:0x0] 00000008:00010000:3.0:1769074480.269130:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.269136:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76a30, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074480.269159:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.269169:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.269183:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.269189:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.269197:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.269209:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.269225:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x74:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074480.269231:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.269234:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.269238:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76a30 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.269243:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.269245:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76a30 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.269259:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x74:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.269488:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.269501:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.269509:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269517:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.269525:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.269529:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.269532:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80335 00000080:00010000:3.0:1769074480.269536:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80335 from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.269539:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269546:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.269554:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.269563:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-74, flags=0 00000080:00200000:3.0:1769074480.269567:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.269573:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.269579:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269591:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.269596:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.269600:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.269603:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80335 00000080:00010000:3.0:1769074480.269606:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80335 from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.269609:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269615:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.269622:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.269629:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-74, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.269693:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269731:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.269744:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x74:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.269975:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.269985:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.269991:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.269995:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005654a00/0x68a7113811d7f9f7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x74:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed73 expref: -99 pid: 30809 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.270021:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80335 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30623c1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.270044:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.270057:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dca80 x1855007612344320/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.270089:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344320:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.270319:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.276133:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344320:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.276240:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x74:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.276262:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.276497:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x74:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.276571:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x74:0x0](ffff8b1004819988) 00000008:00010000:3.0:1769074480.276620:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.276766:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.277548:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.277693:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.277704:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.277718:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.277757:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344448:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.279308:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.284156:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344448:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.284198:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284215:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284240:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284254:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284270:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284279:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284288:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.284370:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dd880 x1855007612344448/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.284387:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284404:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.284413:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.284452:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.284460:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8033c 00000080:00010000:3.0:1769074480.284467:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.284471:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8033c from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.284474:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.284484:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.284514:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.284530:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-75, flags=8 00000080:00200008:3.0:1769074480.284546:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-75:[0x200001b71:0x75:0x0](ffff8b100481f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.284552:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x75:0x0](ffff8b100481f448),name=f5.replay-single-75 00000002:00010000:3.0:1769074480.284575:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x75:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.284586:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.284599:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.284607:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.284610:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x75:0x0](ffff8b100481f448) 00000080:00010000:3.0:1769074480.284614:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x75:0x0](ffff8b100481f448) for lock 0x68a7113811d7f9fe 00000080:00010000:3.0:1769074480.284618:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f9fe from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.284622:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.284629:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.285404:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x75:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.285424:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x75:0x0] 00000008:00010000:3.0:1769074480.285452:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.285457:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76830, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074480.285482:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.285495:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.285509:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.285513:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.285521:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.285535:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.285551:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x75:0x0] updating i_size 7 i_blocks 11 00000020:00010000:3.0:1769074480.285557:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.285561:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.285564:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76830 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.285569:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.285573:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76830 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.285588:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x75:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.285631:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.286186:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.286201:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.286214:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.286222:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.286436:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.286440:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8033c 00000080:00010000:3.0:1769074480.286447:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8033c from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.286451:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.286458:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.286473:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.286489:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-75, flags=0 00000080:00200000:3.0:1769074480.286496:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.286506:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.286513:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.286521:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.286533:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.286536:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.286539:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8033c 00000080:00010000:3.0:1769074480.286543:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8033c from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.286547:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.286553:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.286560:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.286566:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-75, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.286601:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.286892:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.286908:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x75:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.287105:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.287115:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.287121:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.287124:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655e00/0x68a7113811d7f9fe lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ed96 expref: -99 pid: 30810 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.287153:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d8033c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30623eb expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.287177:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.287194:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dd880 x1855007612344704/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.287392:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344704:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.287608:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.299565:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344704:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.299733:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x75:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.299753:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.299923:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x75:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.300125:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x75:0x0](ffff8b100481f448) 00000008:00010000:3.0:1769074480.300168:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 11 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 11 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.300275:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.300301:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.300378:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.300387:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.300399:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.300428:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344832:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.300511:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.305978:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612344832:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.306017:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306031:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306045:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306057:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306071:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306080:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306087:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.306100:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc380 x1855007612344832/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.306116:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306133:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.306141:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.306180:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.306188:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80343 00000080:00010000:3.0:1769074480.306194:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.306198:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80343 from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.306201:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.306211:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.306253:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.306269:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-76, flags=8 00000080:00200008:3.0:1769074480.306286:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-76:[0x200001b71:0x76:0x0](ffff8b100481ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.306419:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x76:0x0](ffff8b100481ba88),name=f5.replay-single-76 00000002:00010000:3.0:1769074480.306449:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x76:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.306463:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.306475:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.306482:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.306488:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x76:0x0](ffff8b100481ba88) 00000080:00010000:3.0:1769074480.306491:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x76:0x0](ffff8b100481ba88) for lock 0x68a7113811d7fa05 00000080:00010000:3.0:1769074480.306496:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa05 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.306499:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.306506:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.306533:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x76:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.306547:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x76:0x0] 00000008:00010000:3.0:1769074480.306573:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.306577:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76830, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074480.306599:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.306610:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.306625:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.306629:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.306677:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.306692:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.306707:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x76:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074480.306712:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.306716:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.306719:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76830 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.306724:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.306728:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76830 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.306742:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x76:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.307200:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.307216:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.307236:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.307247:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.307254:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.307259:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.307262:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80343 00000080:00010000:3.0:1769074480.307266:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80343 from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.307269:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.307276:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.307287:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.308291:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-76, flags=0 00000080:00200000:0.0:1769074480.308309:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.308336:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.308353:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.308368:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.308375:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.308385:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.308389:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80343 00000080:00010000:0.0:1769074480.308396:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80343 from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.308400:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.308409:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.308428:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.308437:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-76, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.308475:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.308520:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.308531:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x76:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.308572:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.308580:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.308585:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.308588:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10057c5400/0x68a7113811d7fa05 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305edb9 expref: -99 pid: 30811 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.308609:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80343 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062415 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.308629:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.308689:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c1c00 x1855007612345088/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.308713:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612345088:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.308909:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2900 going to sleep for 11 seconds 00010000:00010000:0.0:1769074480.344934:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.344954:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.344966:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.344970:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80351 00000080:00010000:0.0:1769074480.344979:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80351 from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.344982:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.344992:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.345007:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.345027:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-78, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.345073:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.345112:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565ea00/0x68a7113811d7fa13 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x78:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305edff expref: -99 pid: 30813 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.345125:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x78:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.345176:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565ea00/0x68a7113811d7fa13 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x78:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305edff expref: -99 pid: 30813 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.345184:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.345190:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.345193:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565ea00/0x68a7113811d7fa13 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x78:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305edff expref: -99 pid: 30813 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.345214:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698200/0x68a7113811d80351 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062469 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.345473:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.345487:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c3b80 x1855007612345856/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.345516:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612345856:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.345626:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.353475:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3b80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612345856:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.353566:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x78:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074480.353585:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.353623:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x78:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.353752:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x78:0x0](ffff8b1018191148) 00000008:00010000:0.0:1769074480.353798:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.353909:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.353934:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.354030:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.354040:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.354052:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.354079:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612345984:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.354162:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.360131:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612345984:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.360174:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360190:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360205:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360218:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360241:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360251:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360260:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.360274:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c0000 x1855007612345984/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.360301:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360318:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.360326:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.360369:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.360377:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80358 00000080:00010000:0.0:1769074480.360383:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.360387:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80358 from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.360391:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.360401:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.360434:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.360450:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-79, flags=8 00000080:00200008:0.0:1769074480.360468:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-79:[0x200001b71:0x79:0x0](ffff8b100481a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074480.360474:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x79:0x0](ffff8b100481a1c8),name=f5.replay-single-79 00000002:00010000:0.0:1769074480.360499:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x79:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.360512:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.360524:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.360531:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.360535:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x79:0x0](ffff8b100481a1c8) 00000080:00010000:0.0:1769074480.360539:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x79:0x0](ffff8b100481a1c8) for lock 0x68a7113811d7fa1a 00000080:00010000:0.0:1769074480.360544:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa1a from it ffffaa914211fcb8 00010000:00010000:0.0:1769074480.360548:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.360556:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.360583:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x79:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074480.360599:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x79:0x0] 00000008:00010000:0.0:1769074480.360627:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.360632:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d130, osc lock ffff8b10097a2390, flags 0x400001002 00010000:00010000:0.0:1769074480.362445:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.362598:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.362618:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.362622:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.362631:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.362817:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.362838:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x79:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074480.362845:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.362849:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.362852:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d130 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.362857:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.362860:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d130 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.362910:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x79:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.363214:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.363253:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.363267:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363279:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.363287:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.363451:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.363456:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80358 00000080:00010000:1.0:1769074480.363464:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80358 from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.363467:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363475:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.363489:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.363502:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-79, flags=0 00000080:00200000:1.0:1769074480.363509:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.363516:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.363522:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363530:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.363536:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.363549:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.363551:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80358 00000080:00010000:1.0:1769074480.363556:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80358 from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.363559:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363565:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.363573:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.363581:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-79, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.363618:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363786:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.363801:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x79:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.363853:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.363861:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.363867:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.363870:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a771400/0x68a7113811d7fa1a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x79:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee22 expref: -99 pid: 30814 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.363892:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d80358 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062493 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.363927:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.363945:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042a00 x1855007612346240/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.364186:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346240:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.364492:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.373284:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346240:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.373379:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x79:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074480.373397:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.373433:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x79:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.373503:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x79:0x0](ffff8b100481a1c8) 00000008:00010000:0.0:1769074480.373547:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.375889:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.375949:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.376425:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.376439:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.376451:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.376489:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346368:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.376620:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074480.383953:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346368:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074480.384176:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384191:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384209:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384223:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384403:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384412:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384419:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074480.384462:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c0000 x1855007612346368/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074480.384477:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384493:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074480.384502:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074480.384538:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074480.384545:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8035f 00000080:00010000:1.0:1769074480.384552:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.384555:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8035f from it ffffaa914211fb58 00010000:00010000:1.0:1769074480.384558:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.384567:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.384600:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.384618:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-8, flags=8 00000080:00200008:1.0:1769074480.384635:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-8:[0x200001b71:0x32:0x0](ffff8b1018123248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074480.386124:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x32:0x0](ffff8b1018123248),name=f5.replay-single-8 00000002:00010000:1.0:1769074480.386168:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x32:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.386187:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.386205:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.386387:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.386399:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x32:0x0](ffff8b1018123248) 00000080:00010000:1.0:1769074480.386404:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x32:0x0](ffff8b1018123248) for lock 0x68a7113811d7f7f1 00000080:00010000:1.0:1769074480.386413:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7f1 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074480.386417:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.386425:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.386460:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x32:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074480.386476:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x32:0x0] 00000008:00010000:1.0:1769074480.386502:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074480.386507:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2030, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074480.386531:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.386542:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.386555:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.386559:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.386567:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.386589:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.386605:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x32:0x0] updating i_size 6 i_blocks 1 00000020:00010000:1.0:1769074480.386611:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.386614:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.386618:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2030 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.386623:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.386626:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2030 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.387255:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x32:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.387337:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.387358:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.387371:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387384:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.387391:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.387397:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.387400:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8035f 00000080:00010000:1.0:1769074480.387406:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8035f from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.387409:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387415:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.387427:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.387453:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-8, flags=0 00000080:00200000:1.0:1769074480.387458:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.387465:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.387472:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387479:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.387485:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.387489:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.387492:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8035f 00000080:00010000:1.0:1769074480.387495:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8035f from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.387498:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387505:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.387513:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.387519:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-8, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.387551:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387585:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.387596:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x32:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.387694:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.387713:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.387718:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.387721:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170200/0x68a7113811d7f7f1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x32:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e378 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.387745:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d8035f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30624bd expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.387770:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.387796:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006040380 x1855007612346624/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.387828:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346624:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.387951:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074480.396403:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006040380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346624:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074480.396495:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x32:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074480.396517:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074480.396555:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x32:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.396625:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x32:0x0](ffff8b1018123248) 00000008:00010000:1.0:1769074480.396732:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 6 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 6 blocks 1 [cma]time 1769074407 1769074407 1769074465 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074480.398896:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.400709:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.400837:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80366 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.400851:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80366 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.400864:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d80366 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.400903:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612346752:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.401610:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000008:00010000:2.0:1769074480.419085:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.419734:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.419754:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.421272:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.421295:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.421311:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.421350:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347136:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.421445:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.429198:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347136:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.429472:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429489:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429503:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429516:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429531:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429541:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429548:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.429561:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c44000 x1855007612347136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.429578:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429594:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.429602:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.429877:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.429886:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8036d 00000080:00010000:2.0:1769074480.429894:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.429898:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8036d from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.429901:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.429910:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.429941:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.429957:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-81, flags=8 00000080:00200008:2.0:1769074480.429973:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-81:[0x200001b71:0x7b:0x0](ffff8b1018177448), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.429979:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7b:0x0](ffff8b1018177448),name=f5.replay-single-81 00000002:00010000:2.0:1769074480.430009:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x7b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.430021:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.430033:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.430039:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.430043:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7b:0x0](ffff8b1018177448) 00000080:00010000:2.0:1769074480.430047:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7b:0x0](ffff8b1018177448) for lock 0x68a7113811d7fa28 00000080:00010000:2.0:1769074480.430051:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa28 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.430054:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.430060:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.430088:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.430107:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7b:0x0] 00000008:00010000:2.0:1769074480.430136:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.430140:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef9b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.430168:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.430180:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.430194:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.430199:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.430207:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.430219:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.430520:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x7b:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.430526:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.430530:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.430534:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef9b0 (ffff8b1007e9b998/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.430539:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.430543:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef9b0 (ffff8b1007e9b638/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.430556:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.430600:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.430610:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.430618:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.430626:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.430633:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.430762:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.430767:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8036d 00000080:00010000:2.0:1769074480.430773:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8036d from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.430858:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.430867:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.430880:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.430892:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-81, flags=0 00000080:00200000:2.0:1769074480.430897:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.430905:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.430911:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.430920:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.430926:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.430931:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.430933:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8036d 00000080:00010000:2.0:1769074480.430937:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8036d from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.430940:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.430947:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.430954:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.430961:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-81, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.430991:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.431026:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.431043:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x7b:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.431093:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.431101:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.431107:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.431111:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770200/0x68a7113811d7fa28 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee68 expref: -99 pid: 30816 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.431132:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8036d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062511 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.431156:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.431172:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c44000 x1855007612347392/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.431205:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347392:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.431844:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.445892:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347392:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.445998:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.446018:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.446058:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.446141:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7b:0x0](ffff8b1018177448) 00000008:00010000:2.0:1769074480.446188:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.446324:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.446348:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.446419:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.446427:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.446440:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.446467:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347520:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.446574:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.455747:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347520:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.455796:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455812:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455826:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455839:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455853:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455862:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455871:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.455882:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c44a80 x1855007612347520/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.455898:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455913:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.455921:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.455967:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.455974:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80374 00000080:00010000:2.0:1769074480.455980:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.455984:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80374 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.455987:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.455996:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.456026:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.456042:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-82, flags=8 00000080:00200008:2.0:1769074480.456056:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-82:[0x200001b71:0x7c:0x0](ffff8b1018196c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.456062:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7c:0x0](ffff8b1018196c08),name=f5.replay-single-82 00000002:00010000:2.0:1769074480.456086:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x7c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.456098:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.456110:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.456116:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.456121:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7c:0x0](ffff8b1018196c08) 00000080:00010000:2.0:1769074480.456124:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7c:0x0](ffff8b1018196c08) for lock 0x68a7113811d7fa2f 00000080:00010000:2.0:1769074480.456128:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa2f from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.456131:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.456139:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.456165:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.456181:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7c:0x0] 00000008:00010000:2.0:1769074480.456208:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.456213:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034efd30, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.456236:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.456248:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.456263:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.456267:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.456276:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.456289:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.456306:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x7c:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.456313:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.456316:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.456320:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034efd30 (ffff8b1007e9b998/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.456325:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.456328:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034efd30 (ffff8b1007e9b638/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.456342:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.456388:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.456397:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.456404:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456413:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.456419:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.456423:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.456426:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80374 00000080:00010000:2.0:1769074480.456430:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80374 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.456433:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456440:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.456449:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.456458:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-82, flags=0 00000080:00200000:2.0:1769074480.456463:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.456469:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.456474:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456482:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.456488:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.456492:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.456494:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80374 00000080:00010000:2.0:1769074480.456498:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80374 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.456501:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456508:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.456515:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.456521:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-82, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.456548:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456583:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.456599:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x7c:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.456709:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.456718:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.456724:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.456727:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100a770e00/0x68a7113811d7fa2f lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x7c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ee8b expref: -99 pid: 30817 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.456750:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80374 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306253b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.456769:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.456778:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c44a80 x1855007612347776/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.456807:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347776:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.456906:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.473712:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347776:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.474066:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7c:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.474088:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.474131:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.475076:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7c:0x0](ffff8b1018196c08) 00000008:00010000:2.0:1769074480.475129:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.475276:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.475315:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.475934:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.475948:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.475968:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.476005:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347904:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.476125:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.486107:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612347904:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.486148:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486164:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486180:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486195:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486208:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486226:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486234:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.486248:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c45880 x1855007612347904/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.486264:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486281:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.486443:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.486487:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.486497:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8037b 00000080:00010000:2.0:1769074480.486505:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.486509:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8037b from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.486513:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.486523:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.486555:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.486573:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-83, flags=8 00000080:00200008:2.0:1769074480.486591:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-83:[0x200001b71:0x7d:0x0](ffff8b1018193248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.486597:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7d:0x0](ffff8b1018193248),name=f5.replay-single-83 00000002:00010000:2.0:1769074480.486625:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x7d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.486637:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.486698:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.486705:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.486710:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7d:0x0](ffff8b1018193248) 00000080:00010000:2.0:1769074480.486713:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7d:0x0](ffff8b1018193248) for lock 0x68a7113811d7fa36 00000080:00010000:2.0:1769074480.486717:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa36 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.486720:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.486727:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.486752:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.486766:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7d:0x0] 00000008:00010000:1.0:1769074480.491706:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074480.491740:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2030, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074480.492016:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.492038:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.492055:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.492061:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.492070:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.492089:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.492108:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x7d:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074480.492114:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.492118:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.492121:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2030 (ffff8b10056856c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.492125:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80000 (ffff8b1007e9b638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.492128:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2030 (ffff8b1007e9b638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.492148:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7d:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.492391:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.492417:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.492430:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.492440:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.492458:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.492467:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.492471:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8037b 00000080:00010000:1.0:1769074480.492478:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8037b from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.492481:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.492488:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.492500:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.492513:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-83, flags=0 00000080:00200000:1.0:1769074480.492519:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.492526:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.492532:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.492539:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.492545:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.492548:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.492551:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8037b 00000080:00010000:1.0:1769074480.492555:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8037b from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.492557:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.492564:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.492580:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.492587:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-83, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.492623:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.493034:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.493051:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x7d:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.493104:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.493112:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.493118:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.493120:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619c00/0x68a7113811d7fa36 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x7d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eeae expref: -99 pid: 30818 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.493142:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d8037b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062565 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.493163:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.493180:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006042d80 x1855007612348160/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.493211:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348160:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.493419:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec0c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.510030:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006042d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348160:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074480.510129:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074480.510149:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074480.510187:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.510260:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7d:0x0](ffff8b1018193248) 00000008:00010000:0.0:1769074480.510305:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.510414:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.510437:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.510513:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.510523:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.510539:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.510564:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348288:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.510724:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2f00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074480.521791:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348288:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074480.521849:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521864:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521878:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521891:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521903:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521911:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521917:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074480.521928:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c3100 x1855007612348288/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074480.521944:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.521959:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074480.521968:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074480.522007:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074480.522023:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80382 00000080:00010000:1.0:1769074480.522031:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.522034:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80382 from it ffffaa914211fb58 00010000:00010000:1.0:1769074480.522038:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522047:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.522081:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.522096:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-84, flags=8 00000080:00200008:1.0:1769074480.522113:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-84:[0x200001b71:0x7e:0x0](ffff8b1018195348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074480.522120:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7e:0x0](ffff8b1018195348),name=f5.replay-single-84 00000002:00010000:1.0:1769074480.522149:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x7e:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.522162:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.522175:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.522181:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.522186:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7e:0x0](ffff8b1018195348) 00000080:00010000:1.0:1769074480.522189:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7e:0x0](ffff8b1018195348) for lock 0x68a7113811d7fa3d 00000080:00010000:1.0:1769074480.522193:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa3d from it ffffaa914211fcb8 00010000:00010000:1.0:1769074480.522196:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.522203:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.522245:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074480.522260:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7e:0x0] 00000008:00010000:1.0:1769074480.522288:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074480.522292:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2030, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074480.522315:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.522325:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.522338:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.522343:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.522350:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.522360:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.522378:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x7e:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074480.522383:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.522386:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.522390:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2030 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.522394:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.522397:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2030 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.522411:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7e:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.522457:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.522466:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.522495:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522504:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.522510:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.522515:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.522517:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80382 00000080:00010000:1.0:1769074480.522521:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80382 from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.522524:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522530:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.522540:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.522549:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-84, flags=0 00000080:00200000:1.0:1769074480.522555:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.522561:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.522567:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522574:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.522579:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.522582:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.522584:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80382 00000080:00010000:1.0:1769074480.522588:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80382 from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.522596:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522602:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.522610:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.522618:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-84, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.522696:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522731:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.522743:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x7e:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.522797:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.522806:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.522812:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.522816:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618e00/0x68a7113811d7fa3d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x7e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eed1 expref: -99 pid: 30819 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.522839:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad400/0x68a7113811d80382 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306258f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.522866:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.522878:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006137480 x1855007612348544/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.522920:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348544:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.523029:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074480.528760:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348544:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074480.528868:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7e:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074480.528887:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074480.528928:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.529006:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7e:0x0](ffff8b1018195348) 00000008:00010000:1.0:1769074480.529052:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074480.529166:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.529191:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.529295:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074480.529306:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074480.529317:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074480.529344:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348672:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074480.529433:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec840 going to sleep for 11 seconds 00000100:00100000:1.0:1769074480.534548:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348672:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074480.534584:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534624:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534681:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534696:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534711:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534719:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534726:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074480.534737:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134700 x1855007612348672/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074480.534754:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.534770:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074480.534933:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074480.535036:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:1.0:1769074480.535046:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80389 00000080:00010000:1.0:1769074480.535053:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.535063:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80389 from it ffffaa914211fb58 00010000:00010000:1.0:1769074480.535066:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.535075:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.535112:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.535127:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-85, flags=8 00000080:00200008:1.0:1769074480.535144:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-85:[0x200001b71:0x7f:0x0](ffff8b1004826c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074480.535150:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7f:0x0](ffff8b1004826c08),name=f5.replay-single-85 00000002:00010000:1.0:1769074480.535174:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x7f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.535186:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.535198:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.535203:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.535208:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7f:0x0](ffff8b1004826c08) 00000080:00010000:1.0:1769074480.535211:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7f:0x0](ffff8b1004826c08) for lock 0x68a7113811d7fa44 00000080:00010000:1.0:1769074480.535215:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa44 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074480.535229:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.535237:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.535262:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1769074480.536363:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7f:0x0] 00000008:00010000:1.0:1769074480.536404:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074480.536410:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2830, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074480.536438:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.536454:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.536469:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.536473:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.536482:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.536496:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.536512:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x7f:0x0] updating i_size 7 i_blocks 1 00000020:00010000:1.0:1769074480.536517:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.536521:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.536524:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2830 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.536529:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.536532:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2830 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.536549:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7f:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.536600:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.536619:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.536630:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.536693:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.536700:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.536707:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.536710:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80389 00000080:00010000:1.0:1769074480.536715:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80389 from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.536718:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.536724:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.536737:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.536750:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-85, flags=0 00000080:00200000:1.0:1769074480.536756:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.536762:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.536767:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.537540:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.537555:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.537562:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.537565:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80389 00000080:00010000:1.0:1769074480.537570:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80389 from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.537573:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.537592:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.537604:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.537612:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-85, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.537716:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.537755:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.537768:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x7f:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.538406:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.538417:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.538424:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.538427:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618600/0x68a7113811d7fa44 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x7f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305eef4 expref: -99 pid: 30820 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.538455:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80389 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30625b9 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.538478:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.538492:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006134700 x1855007612348928/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.538540:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348928:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.538791:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.550332:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612348928:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.550441:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x7f:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.550463:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.550509:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x7f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.550590:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7f:0x0](ffff8b1004826c08) 00000008:00010000:2.0:1769074480.550702:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.550881:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.550905:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.550994:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.551003:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.551015:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.551045:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349056:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.551136:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.555894:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349056:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.555935:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.555952:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556172:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556188:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556205:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556282:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556379:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.556394:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c45880 x1855007612349056/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.556412:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556429:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.556438:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.556480:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.556488:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80390 00000080:00010000:2.0:1769074480.556496:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.556500:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80390 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.556503:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.556512:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.556545:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.556559:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-86, flags=8 00000080:00200008:2.0:1769074480.556575:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-86:[0x200001b71:0x80:0x0](ffff8b1004821148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.556581:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x80:0x0](ffff8b1004821148),name=f5.replay-single-86 00000002:00010000:2.0:1769074480.556608:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x80:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.556621:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.556634:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.556691:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.556696:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x80:0x0](ffff8b1004821148) 00000080:00010000:2.0:1769074480.556699:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x80:0x0](ffff8b1004821148) for lock 0x68a7113811d7fa4b 00000080:00010000:2.0:1769074480.556704:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa4b from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.556707:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.556714:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.556744:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x80:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.556759:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x80:0x0] 00000008:00010000:2.0:1769074480.556947:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.556952:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034efd30, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.557201:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.557215:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.557393:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.557397:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.557405:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.557419:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.557438:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x80:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.557443:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.557446:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.557449:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034efd30 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.557454:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.557457:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034efd30 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.557473:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x80:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.557519:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.557532:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.557541:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557551:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.557557:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.557562:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.557565:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80390 00000080:00010000:2.0:1769074480.557570:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80390 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.557573:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557580:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.557590:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.557602:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-86, flags=0 00000080:00200000:2.0:1769074480.557608:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.557615:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.557620:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557629:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.557635:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.557688:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.557692:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80390 00000080:00010000:2.0:1769074480.557696:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80390 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.557699:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557706:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.557714:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.557721:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-86, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.557756:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557874:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.557889:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x80:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.557934:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.557947:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.557954:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.557957:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619e00/0x68a7113811d7fa4b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x80:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef17 expref: -99 pid: 30821 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.558439:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80390 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30625e3 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.558466:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.558482:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c45880 x1855007612349312/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.558515:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349312:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.558800:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.568414:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349312:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.568517:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x80:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.568537:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.568576:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x80:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.571441:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x80:0x0](ffff8b1004821148) 00000008:00010000:2.0:1769074480.571507:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.572065:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.572097:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.572194:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.572205:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.572434:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.572476:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349440:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.572692:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.580124:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349440:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.580160:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580175:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580188:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580201:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580214:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580230:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580239:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.580254:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c44e00 x1855007612349440/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.580269:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580295:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.580304:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.580341:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.580349:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80397 00000080:00010000:2.0:1769074480.580356:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.580359:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80397 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.580363:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580372:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.580403:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.580417:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-87, flags=8 00000080:00200008:2.0:1769074480.580433:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-87:[0x200001b71:0x81:0x0](ffff8b1004820908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.580439:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x81:0x0](ffff8b1004820908),name=f5.replay-single-87 00000002:00010000:2.0:1769074480.580465:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x81:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.580477:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.580488:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.580496:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.580500:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x81:0x0](ffff8b1004820908) 00000080:00010000:2.0:1769074480.580503:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x81:0x0](ffff8b1004820908) for lock 0x68a7113811d7fa52 00000080:00010000:2.0:1769074480.580508:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa52 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.580511:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.580518:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.580542:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x81:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.580557:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x81:0x0] 00000008:00010000:2.0:1769074480.580582:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.580586:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef2b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.580611:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.580621:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.580635:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.580686:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.580695:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.580711:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.580729:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x81:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.580735:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.580739:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.580743:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef2b0 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.580747:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.580750:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef2b0 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.580763:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x81:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.580954:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.580965:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.580972:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.580982:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.580988:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.580993:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.580996:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80397 00000080:00010000:2.0:1769074480.581001:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80397 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.581003:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.581010:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.581020:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.581029:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-87, flags=0 00000080:00200000:2.0:1769074480.581034:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.581041:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.581048:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.581055:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.581061:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.581065:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.581068:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80397 00000080:00010000:2.0:1769074480.581071:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80397 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.581074:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.581080:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.581087:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.581093:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-87, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.581120:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.581151:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.581163:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x81:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.581215:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.581229:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.581236:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.581239:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655200/0x68a7113811d7fa52 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x81:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef3a expref: -99 pid: 30822 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.581267:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d80397 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306260d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.581355:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.581368:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c44e00 x1855007612349696/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.581397:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349696:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.581510:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.587080:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349696:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.587175:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x81:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.587194:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.587231:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x81:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.587303:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x81:0x0](ffff8b1004820908) 00000008:00010000:2.0:1769074480.587350:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.587463:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.587490:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.587579:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.587588:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.587599:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.587626:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349824:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.587762:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.598091:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612349824:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.598132:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598147:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598162:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598174:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598188:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598198:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598206:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.598230:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c46680 x1855007612349824/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.598245:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598261:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.598269:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.598322:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.598329:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8039e 00000080:00010000:2.0:1769074480.598336:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.598339:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8039e from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.598343:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598352:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.598386:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.598402:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-88, flags=8 00000080:00200008:2.0:1769074480.598422:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-88:[0x200001b71:0x82:0x0](ffff8b100481b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.598429:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x82:0x0](ffff8b100481b248),name=f5.replay-single-88 00000002:00010000:2.0:1769074480.598461:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x82:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.598472:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.598485:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.598492:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.598497:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x82:0x0](ffff8b100481b248) 00000080:00010000:2.0:1769074480.598500:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x82:0x0](ffff8b100481b248) for lock 0x68a7113811d7fa59 00000080:00010000:2.0:1769074480.598503:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa59 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.598506:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.598513:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.598538:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x82:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.598552:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x82:0x0] 00000008:00010000:2.0:1769074480.598577:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.598581:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef2b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.598603:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.598612:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.598625:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.598629:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.598689:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.598707:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.598722:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x82:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.598727:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.598729:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.598733:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef2b0 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.598737:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.598740:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef2b0 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.598753:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x82:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.598807:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.598815:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.598821:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598829:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.598834:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.598838:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.598841:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8039e 00000080:00010000:2.0:1769074480.598845:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8039e from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.598847:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598854:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.598862:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.598871:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-88, flags=0 00000080:00200000:2.0:1769074480.598876:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.598882:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.598888:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598896:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.598902:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.598905:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.598908:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8039e 00000080:00010000:2.0:1769074480.598912:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8039e from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.598915:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598921:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.598928:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.598934:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-88, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.598963:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.598993:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.599004:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x82:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.599053:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.599065:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.599070:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.599073:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618c00/0x68a7113811d7fa59 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef5d expref: -99 pid: 30823 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.599094:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8039e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062637 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.599115:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.599125:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c46680 x1855007612350080/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.599153:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350080:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.599266:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.606490:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350080:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.606577:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x82:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.606594:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.606633:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x82:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.606817:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x82:0x0](ffff8b100481b248) 00000008:00010000:2.0:1769074480.606859:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074409 1769074409 0 by oinfo size 7 blocks 1 [cma]time 1769074409 1769074409 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.607067:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.607095:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.607171:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.607180:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.607192:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.607218:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350208:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.607405:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.613384:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350208:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.613419:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613433:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613447:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613459:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613472:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613480:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613487:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.613499:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c45180 x1855007612350208/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.613514:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613530:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.613538:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.613577:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.613585:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803a5 00000080:00010000:2.0:1769074480.613591:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.613594:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803a5 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.613597:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.613606:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.613635:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.613686:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-89, flags=8 00000080:00200008:2.0:1769074480.613704:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-89:[0x200001b71:0x83:0x0](ffff8b100481d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.613710:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x83:0x0](ffff8b100481d348),name=f5.replay-single-89 00000002:00010000:2.0:1769074480.613734:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x83:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.613747:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.613760:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.613768:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.613785:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x83:0x0](ffff8b100481d348) 00000080:00010000:2.0:1769074480.613788:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x83:0x0](ffff8b100481d348) for lock 0x68a7113811d7fa60 00000080:00010000:2.0:1769074480.613793:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa60 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.613796:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.613804:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.613827:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x83:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.613841:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x83:0x0] 00000008:00010000:2.0:1769074480.613865:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.613870:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034efdb0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.613892:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.613902:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.613916:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.613920:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.613927:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.613944:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.613972:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x83:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.613977:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.613981:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.613984:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034efdb0 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.613989:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.613992:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034efdb0 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.614006:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x83:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.614046:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.614054:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.614061:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614070:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.614076:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.614080:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.614083:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803a5 00000080:00010000:2.0:1769074480.614087:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803a5 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.614090:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614097:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.614106:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.614114:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-89, flags=0 00000080:00200000:2.0:1769074480.614119:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.614125:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.614131:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614139:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.614155:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.614159:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.614162:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803a5 00000080:00010000:2.0:1769074480.614166:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803a5 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.614169:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614177:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.614184:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.614190:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-89, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.614220:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614254:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.614266:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x83:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.614322:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.614335:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.614341:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.614344:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619200/0x68a7113811d7fa60 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x83:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305ef80 expref: -99 pid: 30824 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.614364:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803a5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062661 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.614385:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.614396:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c45180 x1855007612350464/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.614426:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350464:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.614554:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.622000:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350464:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.622092:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x83:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.622111:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.622631:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x83:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.622760:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x83:0x0](ffff8b100481d348) 00000008:00010000:0.0:1769074480.623472:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074480.623593:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.623619:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.623882:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.623892:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:0.0:1769074480.623904:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:0.0:1769074480.623939:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350592:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074480.624636:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.630420:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350592:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074480.630459:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630474:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630494:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630506:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630520:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630529:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630537:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074480.630549:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10056c2680 x1855007612350592/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074480.630566:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.630581:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074480.630588:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074480.630627:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:0.0:1769074480.630634:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ac 00000080:00010000:0.0:1769074480.632916:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.632927:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ac from it ffffaa914211fb58 00010000:00010000:0.0:1769074480.632935:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.632951:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.633140:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.633381:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-9, flags=8 00000080:00200008:1.0:1769074480.633574:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-9:[0x200001b71:0x33:0x0](ffff8b10181221c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074480.633582:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x33:0x0](ffff8b10181221c8),name=f5.replay-single-9 00000002:00010000:1.0:1769074480.633616:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x33:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.633634:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.633926:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.633934:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.633944:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x33:0x0](ffff8b10181221c8) 00000080:00010000:1.0:1769074480.633948:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x33:0x0](ffff8b10181221c8) for lock 0x68a7113811d7f7f8 00000080:00010000:1.0:1769074480.633953:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7f7f8 from it ffffaa914211fcb8 00010000:00010000:1.0:1769074480.633957:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.633965:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.633996:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x33:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1769074480.634011:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x33:0x0] 00000008:00010000:1.0:1769074480.634038:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074480.634042:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10031f2030, osc lock ffff8b10030dee40, flags 0x400001002 00010000:00010000:1.0:1769074480.634064:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.634096:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.634109:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074480.634113:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074480.634121:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074480.634132:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074480.634158:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x33:0x0] updating i_size 6 i_blocks 1 00000020:00010000:1.0:1769074480.634164:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074480.634167:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.634170:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10031f2030 (ffff8b1005685a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074480.634174:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768490 (ffff8b10056856c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074480.634177:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10031f2030 (ffff8b10056856c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074480.634190:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x33:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.634248:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.634258:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.634266:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634273:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.634279:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.634371:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.634373:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ac 00000080:00010000:1.0:1769074480.634378:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ac from it ffffaa914211fc10 00010000:00010000:1.0:1769074480.634381:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634388:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.634397:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.634407:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-9, flags=0 00000080:00200000:1.0:1769074480.634412:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.634419:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.634424:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634431:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.634436:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.634440:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.634442:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ac 00000080:00010000:1.0:1769074480.634445:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ac from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.634447:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634453:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.634460:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.634473:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-9, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.634504:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634533:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.634545:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x33:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.634590:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.634597:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.634601:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.634604:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006170c00/0x68a7113811d7f7f8 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x33:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305e394 expref: -99 pid: 30737 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.634625:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d803ac lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306268b expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.635081:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.635100:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135f80 x1855007612350848/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.635133:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350848:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.635570:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ece40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.644538:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350848:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.644631:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x33:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074480.644934:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.645095:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x33:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.645404:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x33:0x0](ffff8b10181221c8) 00000008:00010000:2.0:1769074480.645454:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 6 blocks 1 [cma]time: 1769074407 1769074407 0 by oinfo size 6 blocks 1 [cma]time 1769074407 1769074407 1769074465 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.645573:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.645599:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.646580:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.646595:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.646606:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.646709:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c47800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350976:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.646910:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.652102:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c47800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612350976:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.652140:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652242:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652256:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652269:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652433:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652445:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652453:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.652466:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c47800 x1855007612350976/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.652483:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652500:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.652508:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.652546:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.652553:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803b3 00000080:00010000:2.0:1769074480.652559:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.652562:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803b3 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.652566:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.652575:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.652606:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.652622:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-90, flags=8 00000080:00200008:2.0:1769074480.652868:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-90:[0x200001b71:0x84:0x0](ffff8b100481cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.652876:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x84:0x0](ffff8b100481cb08),name=f5.replay-single-90 00000002:00010000:2.0:1769074480.652901:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x84:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.652912:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.652924:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.652930:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.652935:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x84:0x0](ffff8b100481cb08) 00000080:00010000:2.0:1769074480.652938:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x84:0x0](ffff8b100481cb08) for lock 0x68a7113811d7fa67 00000080:00010000:2.0:1769074480.652942:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa67 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.652945:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.652953:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.653059:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x84:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.653073:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x84:0x0] 00000008:00010000:2.0:1769074480.653098:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.653102:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef3b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.653122:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.653133:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.653370:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.653376:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.653386:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.653399:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.653418:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x84:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.653423:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.653426:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.653430:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef3b0 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.653434:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.653438:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef3b0 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.653452:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x84:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.653496:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.653506:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.653513:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.653522:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.653529:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.653534:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.653536:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803b3 00000080:00010000:2.0:1769074480.653542:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803b3 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.653545:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.653552:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.653561:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.653569:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-90, flags=0 00000080:00200000:2.0:1769074480.653574:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.653580:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.653585:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.653593:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.653598:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.653602:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.653604:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803b3 00000080:00010000:2.0:1769074480.653608:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803b3 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.653611:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.653617:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.653625:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.653631:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-90, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.653953:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.654413:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.654432:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x84:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.654483:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.654490:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.654496:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.654498:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e000/0x68a7113811d7fa67 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x84:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efa3 expref: -99 pid: 30825 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.654522:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803b3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30626b5 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.654541:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.654556:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c47800 x1855007612351232/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.654585:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c47800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351232:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.655009:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.661855:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c47800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351232:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.661954:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x84:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.661984:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.662024:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x84:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.662113:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x84:0x0](ffff8b100481cb08) 00000008:00010000:2.0:1769074480.662171:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074469 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.662321:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.662354:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.662444:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.662453:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.662465:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.662489:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351360:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.662576:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.670175:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351360:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.670212:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670235:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670248:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670259:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670272:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670280:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670470:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.670484:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c46a00 x1855007612351360/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.670501:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670516:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.670524:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.670563:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.670570:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ba 00000080:00010000:2.0:1769074480.670578:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.670581:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ba from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.670585:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.670593:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.670622:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.670636:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-91, flags=8 00000080:00200008:2.0:1769074480.670700:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-91:[0x200001b71:0x85:0x0](ffff8b100481db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.670706:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x85:0x0](ffff8b100481db88),name=f5.replay-single-91 00000002:00010000:2.0:1769074480.670733:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x85:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.670745:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.670756:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.670763:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.670767:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x85:0x0](ffff8b100481db88) 00000080:00010000:2.0:1769074480.670770:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x85:0x0](ffff8b100481db88) for lock 0x68a7113811d7fa6e 00000080:00010000:2.0:1769074480.672074:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa6e from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.672080:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.672092:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.672124:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x85:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.672139:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x85:0x0] 00010000:00010000:3.0:1769074480.672550:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:0.0:1769074480.673572:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074480.673580:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972dd30, osc lock ffff8b10097a2f70, flags 0x400001002 00010000:00010000:0.0:1769074480.673616:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.673636:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.673805:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074480.673810:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074480.673819:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074480.673834:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074480.673852:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x85:0x0] updating i_size 7 i_blocks 1 00000020:00010000:0.0:1769074480.673858:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_release():245 00000020:00010000:0.0:1769074480.673861:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.673864:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972dd30 (ffff8b1007c69878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074480.673869:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f80db0 (ffff8b1007e9b998/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074480.673872:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972dd30 (ffff8b1007e9b998/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074480.673891:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x85:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074480.673944:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.674074:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.674088:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674098:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.674105:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.674114:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.674119:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ba 00000080:00010000:0.0:1769074480.674124:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ba from it ffffaa914211fc10 00010000:00010000:0.0:1769074480.674127:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674135:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.674156:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074480.674168:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-91, flags=0 00000080:00200000:0.0:1769074480.674174:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074480.674181:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074480.674187:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674195:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.674207:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074480.674210:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074480.674213:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803ba 00000080:00010000:0.0:1769074480.674542:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803ba from it ffffaa914211fd68 00010000:00010000:0.0:1769074480.674546:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674554:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074480.674564:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074480.674571:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-91, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074480.674608:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674674:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074480.674687:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x85:0x0]: losing layout lock 00010000:00010000:0.0:1769074480.674736:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.674744:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074480.674749:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074480.674753:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655000/0x68a7113811d7fa6e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x85:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efc6 expref: -99 pid: 30826 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074480.674791:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d803ba lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30626df expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074480.674812:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074480.674828:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10056c2680 x1855007612351616/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074480.674858:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351616:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074480.675488:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074480.686503:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351616:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074480.686728:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x85:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1769074480.686746:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074480.686869:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x85:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074480.687527:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x85:0x0](ffff8b100481db88) 00000008:00010000:1.0:1769074480.687576:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074469 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074480.687827:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.687854:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.687938:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074480.687948:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:1.0:1769074480.687959:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:1.0:1769074480.687989:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351744:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074480.688098:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.695331:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612351744:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.695370:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695386:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695400:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695419:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695433:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695441:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695448:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.695459:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134380 x1855007612351744/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.695474:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695489:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.695498:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.695537:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.695544:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c1 00000080:00010000:2.0:1769074480.695550:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.695553:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c1 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.695557:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.695566:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.695599:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.695615:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-92, flags=8 00000080:00200008:2.0:1769074480.695630:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-92:[0x200001b71:0x86:0x0](ffff8b100481aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.695685:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x86:0x0](ffff8b100481aa08),name=f5.replay-single-92 00000002:00010000:2.0:1769074480.695712:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x86:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.695723:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.695734:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.695741:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.695746:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x86:0x0](ffff8b100481aa08) 00000080:00010000:2.0:1769074480.695749:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x86:0x0](ffff8b100481aa08) for lock 0x68a7113811d7fa75 00000080:00010000:2.0:1769074480.695752:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa75 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.695755:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.695762:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.696073:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x86:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.696089:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x86:0x0] 00000008:00010000:2.0:1769074480.696116:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.696120:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef530, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.696150:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.696166:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.696179:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.696183:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.696190:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.696200:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.696217:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x86:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.696222:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.696225:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.696228:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef530 (ffff8b1007e9b5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.696232:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.696234:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef530 (ffff8b1007e9be18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.696246:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x86:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.696302:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.696312:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.696320:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696328:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.696334:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.696339:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.696341:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c1 00000080:00010000:2.0:1769074480.696346:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c1 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.696349:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696356:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.696365:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.696375:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-92, flags=0 00000080:00200000:2.0:1769074480.696380:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.696386:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.696391:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696398:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.696403:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.696407:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.696410:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c1 00000080:00010000:2.0:1769074480.696413:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c1 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.696416:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696423:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.696430:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.696437:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-92, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.696466:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696494:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.696504:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x86:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.696549:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.696557:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.696563:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.696566:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565e800/0x68a7113811d7fa75 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x86:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305efe9 expref: -99 pid: 30827 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.696589:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d803c1 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062709 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.696611:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.696622:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c46a00 x1855007612352000/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.696709:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352000:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.698131:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.708768:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46a00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352000:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.709091:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x86:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.709110:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.709148:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x86:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.709477:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x86:0x0](ffff8b100481aa08) 00000008:00010000:3.0:1769074480.709526:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.709712:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.709734:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.710955:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.711369:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.711384:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.711418:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352128:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.711539:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.725556:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352128:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.725594:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725610:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725626:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725695:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725711:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725721:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725729:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.725741:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc380 x1855007612352128/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.725757:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725776:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.725785:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.725832:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.725840:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c8 00000080:00010000:3.0:1769074480.725849:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.725854:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c8 from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.725858:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.725867:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.725901:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.726072:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-93, flags=8 00000080:00200008:3.0:1769074480.726091:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-93:[0x200001b71:0x87:0x0](ffff8b100481ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.726098:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x87:0x0](ffff8b100481ec08),name=f5.replay-single-93 00000002:00010000:3.0:1769074480.726125:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x87:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.726136:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.726149:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.726157:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.726162:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x87:0x0](ffff8b100481ec08) 00000080:00010000:3.0:1769074480.726165:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x87:0x0](ffff8b100481ec08) for lock 0x68a7113811d7fa7c 00000080:00010000:3.0:1769074480.726169:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa7c from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.726173:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.726180:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.726208:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x87:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.726234:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x87:0x0] 00000008:00010000:3.0:1769074480.726263:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.726267:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76430, osc lock ffff8b1003616be0, flags 0x400001002 00010000:00010000:3.0:1769074480.727190:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.727436:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.727457:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.727462:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.727470:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.727485:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.727506:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x87:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.727511:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.727514:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.727519:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76430 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.727523:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.727526:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76430 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.727545:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x87:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.727600:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.727618:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.727628:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727655:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.727673:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.727680:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.727683:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c8 00000080:00010000:2.0:1769074480.727689:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c8 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.727692:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727703:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.727713:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.727725:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-93, flags=0 00000080:00200000:2.0:1769074480.727731:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.727738:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.727743:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727750:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.727756:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.727760:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.727763:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803c8 00000080:00010000:2.0:1769074480.727766:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803c8 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.727769:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727777:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.727785:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.727792:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-93, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.727834:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727872:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.727885:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x87:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.727936:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.727944:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.727951:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.727954:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005619a00/0x68a7113811d7fa7c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x87:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f00c expref: -99 pid: 30828 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.728245:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d803c8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062733 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.728269:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.728481:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c47100 x1855007612352384/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.728516:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c47100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352384:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.728779:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.743790:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c47100 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352384:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.744237:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x87:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.744259:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.744304:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x87:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.744371:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x87:0x0](ffff8b100481ec08) 00000008:00010000:2.0:1769074480.744490:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.744621:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.744824:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.744905:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.744915:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.744926:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.744969:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352512:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.745085:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.751295:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352512:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.751333:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751348:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751362:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751374:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751388:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751397:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751404:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.751417:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c46d80 x1855007612352512/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.751432:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751448:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.751457:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.751495:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.751502:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803cf 00000080:00010000:2.0:1769074480.751509:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.751512:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803cf from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.751516:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.751526:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.751559:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.751574:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-94, flags=8 00000080:00200008:2.0:1769074480.751589:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-94:[0x200001b71:0x88:0x0](ffff8b10181763c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.751596:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x88:0x0](ffff8b10181763c8),name=f5.replay-single-94 00000002:00010000:2.0:1769074480.751623:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x88:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.751634:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.751695:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.751702:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.751707:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x88:0x0](ffff8b10181763c8) 00000080:00010000:2.0:1769074480.751710:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x88:0x0](ffff8b10181763c8) for lock 0x68a7113811d7fa83 00000080:00010000:2.0:1769074480.751714:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa83 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.751718:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.751725:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.751751:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x88:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.751764:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x88:0x0] 00000008:00010000:2.0:1769074480.751791:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.752035:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef530, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.752061:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.752073:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.752087:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.752091:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.752101:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.752112:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.752127:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x88:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.752132:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.752136:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.752139:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef530 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.752144:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.752147:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef530 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.752161:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x88:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.752208:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.752218:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.752226:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.752234:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.752240:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.752244:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.752247:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803cf 00000080:00010000:2.0:1769074480.752251:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803cf from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.752255:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.752262:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.752272:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074480.752633:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-94, flags=0 00000080:00200000:1.0:1769074480.752681:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074480.752693:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074480.752701:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.752709:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.752716:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074480.752725:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074480.752728:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803cf 00000080:00010000:1.0:1769074480.752732:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803cf from it ffffaa914211fd68 00010000:00010000:1.0:1769074480.752758:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.752765:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074480.752775:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074480.752783:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-94, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074480.752828:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.752859:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074480.752871:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x88:0x0]: losing layout lock 00010000:00010000:1.0:1769074480.752919:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.752927:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074480.752934:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074480.752937:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f000/0x68a7113811d7fa83 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x88:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f02f expref: -99 pid: 30829 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074480.753102:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d803cf lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306275d expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074480.753127:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074480.753143:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135c00 x1855007612352768/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074480.753186:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352768:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074480.757419:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.768979:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352768:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.769072:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x88:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.769090:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.769128:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x88:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.769199:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x88:0x0](ffff8b10181763c8) 00000008:00010000:3.0:1769074480.769258:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.769529:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.769554:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.769630:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.769686:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.769697:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.769726:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352896:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.770495:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.778232:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612352896:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.778271:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778284:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778312:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778324:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778337:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778347:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778356:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.778369:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc380 x1855007612352896/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.778385:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778402:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.778411:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.778450:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.778458:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803d6 00000080:00010000:3.0:1769074480.778464:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.778468:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803d6 from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.778471:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778480:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.778513:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.778531:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-95, flags=8 00000080:00200008:3.0:1769074480.778549:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-95:[0x200001b71:0x89:0x0](ffff8b100482c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.778555:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x89:0x0](ffff8b100482c2c8),name=f5.replay-single-95 00000002:00010000:3.0:1769074480.778581:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x89:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.778593:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.778606:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.778613:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.778617:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x89:0x0](ffff8b100482c2c8) 00000080:00010000:3.0:1769074480.778620:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x89:0x0](ffff8b100482c2c8) for lock 0x68a7113811d7fa8a 00000080:00010000:3.0:1769074480.778625:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa8a from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.778628:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.778635:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.778712:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x89:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.778731:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x89:0x0] 00000008:00010000:3.0:1769074480.778761:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.778765:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f761b0, osc lock ffff8b10036171d0, flags 0x400001002 00010000:00010000:3.0:1769074480.778790:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.778800:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.778814:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.778818:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.778826:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.778835:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.778853:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x89:0x0] updating i_size 7 i_blocks 11 00000020:00010000:3.0:1769074480.778859:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.778861:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.778865:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f761b0 (ffff8b1005685248/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.778870:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.778873:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f761b0 (ffff8b1007c696c8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.778887:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x89:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.778930:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.778939:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.778946:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778953:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.778959:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.778963:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.778965:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803d6 00000080:00010000:3.0:1769074480.778969:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803d6 from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.778971:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.778978:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.778986:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.778995:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-95, flags=0 00000080:00200000:3.0:1769074480.778999:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.779005:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.779010:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.779018:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.779024:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.779028:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.779030:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803d6 00000080:00010000:3.0:1769074480.779034:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803d6 from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.779037:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.779043:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.779050:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.779056:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-95, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.779082:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.779108:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.779123:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x89:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.779164:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.779172:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.779177:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.779179:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005c3a800/0x68a7113811d7fa8a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x89:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f052 expref: -99 pid: 30830 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.779200:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168200/0x68a7113811d803d6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062787 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.779220:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.779230:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc380 x1855007612353152/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.779256:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353152:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.779354:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.786204:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353152:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.786444:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x89:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.786463:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.786500:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x89:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.786575:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x89:0x0](ffff8b100482c2c8) 00000008:00010000:3.0:1769074480.786620:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 11 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 11 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074480.787151:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.787178:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.787417:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.787428:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:3.0:1769074480.787439:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:3.0:1769074480.787467:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353280:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074480.787690:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.791539:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353280:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074480.791572:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791585:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791600:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791611:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791624:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791632:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791686:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074480.791700:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de300 x1855007612353280/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074480.791718:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791734:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074480.791742:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074480.791784:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:3.0:1769074480.791791:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803dd 00000080:00010000:3.0:1769074480.791798:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.791802:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803dd from it ffffaa914211fb58 00010000:00010000:3.0:1769074480.791806:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.791816:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.791847:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.791862:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-96, flags=8 00000080:00200008:3.0:1769074480.791877:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-96:[0x200001b71:0x8a:0x0](ffff8b100482e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074480.791883:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8a:0x0](ffff8b100482e3c8),name=f5.replay-single-96 00000002:00010000:3.0:1769074480.791908:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x8a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.791921:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.791934:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.791942:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.791947:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8a:0x0](ffff8b100482e3c8) 00000080:00010000:3.0:1769074480.791950:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8a:0x0](ffff8b100482e3c8) for lock 0x68a7113811d7fa91 00000080:00010000:3.0:1769074480.791955:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa91 from it ffffaa914211fcb8 00010000:00010000:3.0:1769074480.792032:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.792041:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.792067:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074480.792081:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8a:0x0] 00000008:00010000:3.0:1769074480.792107:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074480.792111:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f761b0, osc lock ffff8b10036171d0, flags 0x400001002 00010000:00010000:3.0:1769074480.792133:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.792144:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.792159:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074480.792164:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074480.792173:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074480.792187:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074480.792204:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x8a:0x0] updating i_size 7 i_blocks 1 00000020:00010000:3.0:1769074480.792210:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074480.792213:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.792381:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f761b0 (ffff8b1005685248/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074480.792388:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074480.792390:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f761b0 (ffff8b1007c696c8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074480.792405:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8a:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074480.792446:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.792455:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.792463:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792471:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.792478:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.792483:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.792485:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803dd 00000080:00010000:3.0:1769074480.792489:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803dd from it ffffaa914211fc10 00010000:00010000:3.0:1769074480.792493:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792499:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.792509:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074480.792518:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-96, flags=0 00000080:00200000:3.0:1769074480.792524:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074480.792530:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074480.792536:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792544:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.792550:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074480.792554:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074480.792556:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803dd 00000080:00010000:3.0:1769074480.792560:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803dd from it ffffaa914211fd68 00010000:00010000:3.0:1769074480.792562:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792570:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074480.792578:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074480.792584:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-96, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074480.792615:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792780:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074480.792799:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x8a:0x0]: losing layout lock 00010000:00010000:3.0:1769074480.792846:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.792854:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074480.792860:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074480.792863:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005618800/0x68a7113811d7fa91 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f075 expref: -99 pid: 30831 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074480.792885:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d803dd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30627b1 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074480.792907:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074480.792920:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de300 x1855007612353536/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074480.792947:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353536:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074480.793180:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074480.801435:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353536:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074480.801525:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8a:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074480.801544:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074480.801584:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.802061:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8a:0x0](ffff8b100482e3c8) 00000008:00010000:2.0:1769074480.802109:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.802380:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.802404:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.802481:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.802490:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.802501:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.802529:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353664:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.803550:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.806564:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353664:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.806600:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.806614:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.806628:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808564:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808589:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808597:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808604:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.808622:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c46d80 x1855007612353664/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.808761:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808782:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.808791:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.808835:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.808843:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803e4 00000080:00010000:2.0:1769074480.808850:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.808853:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803e4 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.808857:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.808867:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.808900:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.808914:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-97, flags=8 00000080:00200008:2.0:1769074480.808929:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-97:[0x200001b71:0x8b:0x0](ffff8b1004829988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.808935:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8b:0x0](ffff8b1004829988),name=f5.replay-single-97 00000002:00010000:2.0:1769074480.808957:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x8b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.809078:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.809090:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.809098:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.809103:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8b:0x0](ffff8b1004829988) 00000080:00010000:2.0:1769074480.809106:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8b:0x0](ffff8b1004829988) for lock 0x68a7113811d7fa98 00000080:00010000:2.0:1769074480.809110:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa98 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.809114:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.809121:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.809147:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8b:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:2.0:1769074480.809161:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8b:0x0] 00000008:00010000:2.0:1769074480.809188:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.809192:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef3b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.809212:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.809363:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.809533:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.809540:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.809549:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.809561:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.809576:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x8b:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.809582:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.809585:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.809588:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef3b0 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.809593:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.809596:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef3b0 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.809607:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.809740:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.809750:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.809758:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.809768:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.809775:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.809780:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.809783:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803e4 00000080:00010000:2.0:1769074480.809787:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803e4 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.809791:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.809797:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.809807:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.809815:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-97, flags=0 00000080:00200000:2.0:1769074480.809820:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.809824:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.809829:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.809837:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.809844:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.809847:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.809849:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803e4 00000080:00010000:2.0:1769074480.809852:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803e4 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.809855:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.809863:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.809871:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.809878:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-97, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.809906:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.809935:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.809945:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x8b:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.810061:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.810069:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.810074:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.810077:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005655600/0x68a7113811d7fa98 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f098 expref: -99 pid: 30832 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.810099:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803e4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30627db expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.810120:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.810132:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c46d80 x1855007612353920/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.810165:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353920:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.810491:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.816938:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46d80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612353920:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.817184:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.817203:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.817387:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.817462:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8b:0x0](ffff8b1004829988) 00000008:00010000:2.0:1769074480.817508:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.817610:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.817633:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.817815:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.817824:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.817835:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.817861:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354048:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.818122:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.822178:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354048:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.822378:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822395:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822410:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822422:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822437:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822446:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822455:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.822467:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c45500 x1855007612354048/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.822483:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822501:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.822509:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.822550:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.822558:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803eb 00000080:00010000:2.0:1769074480.822566:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.822569:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803eb from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.822573:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.822582:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.822615:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.822632:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-98, flags=8 00000080:00200008:2.0:1769074480.822685:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-98:[0x200001b71:0x8c:0x0](ffff8b100482f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.822692:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8c:0x0](ffff8b100482f448),name=f5.replay-single-98 00000002:00010000:2.0:1769074480.822716:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x8c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.822728:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.822740:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.822747:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.822751:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8c:0x0](ffff8b100482f448) 00000080:00010000:2.0:1769074480.822754:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8c:0x0](ffff8b100482f448) for lock 0x68a7113811d7fa9f 00000080:00010000:2.0:1769074480.822759:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7fa9f from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.822761:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.822768:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.822795:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.822814:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8c:0x0] 00000008:00010000:2.0:1769074480.822844:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.822850:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef3b0, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.822873:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.822885:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.822905:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.822910:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.822918:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.822929:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.822944:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x8c:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.822949:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.822952:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.822955:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef3b0 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.823031:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.823035:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef3b0 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.823048:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.823091:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.823099:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.823106:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823115:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.823121:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.823125:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.823128:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803eb 00000080:00010000:2.0:1769074480.823134:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803eb from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.823137:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823144:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.823152:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.823161:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-98, flags=0 00000080:00200000:2.0:1769074480.823166:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.823173:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.823178:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823187:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.823194:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.823198:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.823200:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803eb 00000080:00010000:2.0:1769074480.823204:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803eb from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.823207:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823214:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.823360:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.823368:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-98, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.823397:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823425:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.823436:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x8c:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.823477:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.823485:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.823491:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.823494:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565f600/0x68a7113811d7fa9f lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f0bb expref: -99 pid: 30833 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.823514:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d803eb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062805 expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.823534:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.823544:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c45500 x1855007612354304/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.823572:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354304:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.823772:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.830449:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45500 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354304:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.830541:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8c:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.830561:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.830598:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.830756:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8c:0x0](ffff8b100482f448) 00000008:00010000:2.0:1769074480.830804:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074480.830929:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.831115:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.831193:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.831201:0:30957:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00010000:00010000:2.0:1769074480.831212:0:30957:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 30957 initiator: MDT0 00000100:00100000:2.0:1769074480.831403:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354432:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074480.831532:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.835774:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354432:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074480.835811:0:30957:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835825:0:30957:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835837:0:30957:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835849:0:30957:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835867:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835876:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835883:0:30957:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074480.835895:0:30957:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005c44700 x1855007612354432/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074496 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074480.835910:0:30957:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.835925:0:30957:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074480.835934:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074480.835983:0:30957:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 24064 00000080:00010000:2.0:1769074480.835992:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f2 00000080:00010000:2.0:1769074480.836000:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.836004:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f2 from it ffffaa914211fb58 00010000:00010000:2.0:1769074480.836007:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836016:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.836048:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.836064:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-99, flags=8 00000080:00200008:2.0:1769074480.836080:0:30957:0:(file.c:6252:ll_getattr_dentry()) START file f5.replay-single-99:[0x200001b71:0x8d:0x0](ffff8b100482ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074480.836086:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8d:0x0](ffff8b100482ba88),name=f5.replay-single-99 00000002:00010000:2.0:1769074480.836110:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x8d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.836122:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.836134:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.836140:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.836145:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8d:0x0](ffff8b100482ba88) 00000080:00010000:2.0:1769074480.836149:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8d:0x0](ffff8b100482ba88) for lock 0x68a7113811d7faa6 00000080:00010000:2.0:1769074480.836153:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d7faa6 from it ffffaa914211fcb8 00010000:00010000:2.0:1769074480.836156:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.836164:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.836189:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074480.836202:0:30957:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8d:0x0] 00000008:00010000:2.0:1769074480.836226:0:30957:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074480.836230:0:30957:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef130, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074480.836252:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.836262:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.836282:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074480.836287:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074480.836295:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074480.836305:0:30957:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074480.836322:0:30957:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x8d:0x0] updating i_size 7 i_blocks 1 00000020:00010000:2.0:1769074480.836328:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074480.836331:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.836334:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef130 (ffff8b1005685128/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074480.836339:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f81240 (ffff8b1007c69e18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074480.836342:0:30957:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef130 (ffff8b1007c69e18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074480.836355:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8d:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.836397:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.836405:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.836412:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836420:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.836427:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.836431:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.836433:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f2 00000080:00010000:2.0:1769074480.836438:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f2 from it ffffaa914211fc10 00010000:00010000:2.0:1769074480.836441:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836449:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.836457:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074480.836465:0:30957:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-99, flags=0 00000080:00200000:2.0:1769074480.836469:0:30957:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074480.836476:0:30957:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074480.836482:0:30957:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836490:0:30957:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.836496:0:30957:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074480.836499:0:30957:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074480.836502:0:30957:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f2 00000080:00010000:2.0:1769074480.836506:0:30957:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f2 from it ffffaa914211fd68 00010000:00010000:2.0:1769074480.836509:0:30957:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836516:0:30957:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074480.836524:0:30957:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074480.836530:0:30957:0:(namei.c:2354:ll_unlink()) VFS Op:name=f5.replay-single-99, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074480.836560:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836586:0:30957:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074480.836597:0:30957:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x8d:0x0]: losing layout lock 00010000:00010000:2.0:1769074480.836686:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.836695:0:30957:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074480.836701:0:30957:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074480.836704:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b100565fa00/0x68a7113811d7faa6 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf305f0de expref: -99 pid: 30834 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074480.836726:0:30957:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d803f2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306282f expref: -99 pid: 30957 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074480.836747:0:30957:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074480.836758:0:30957:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005c44700 x1855007612354688/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074480.836786:0:30957:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354688:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074480.836886:0:30957:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074480.842171:0:30957:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:30957:1855007612354688:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074480.842266:0:30957:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x8d:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:2.0:1769074480.842292:0:30957:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074480.842327:0:30957:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x8d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074480.842396:0:30957:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8d:0x0](ffff8b100482ba88) 00000008:00010000:2.0:1769074480.842438:0:30957:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 7 blocks 1 [cma]time: 1769074410 1769074410 0 by oinfo size 7 blocks 1 [cma]time 1769074410 1769074410 1769074470 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074481.695996:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074482.720131:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074482.849905:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612354816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.850049:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.850091:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1769074482.850098:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c47480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612354944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.850125:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.854987:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c47480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612354944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.855016:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.855027:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612354816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074482.855039:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074483.670338:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.670380:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.670419:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624635520 00000100:00100000:1.0:1769074483.670431:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624635520 00000100:00100000:1.0:1769074483.670481:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624635648 00000100:00100000:1.0:1769074483.670483:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624635648 00000100:00100000:1.0:1769074483.670498:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 102 00000100:00100000:1.0:1769074483.670508:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624635520:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.670528:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.670697:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.670706:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624635520:12345-192.168.204.102@tcp:104: Request processed in 198us (381us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.670715:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 102 00010000:00010000:2.0:1769074483.670718:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1769074483.670738:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 103 00010000:00010000:2.0:1769074483.670739:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.670749:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624635648:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.670756:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d2e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.670761:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670766:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.670797:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.670804:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624635648:12345-192.168.204.102@tcp:104: Request processed in 58us (423us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.670811:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 103 00010000:00010000:2.0:1769074483.670832:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670841:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.670866:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670874:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2e00/0x68a7113811d7f631 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670901:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.670905:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670912:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d2a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.670917:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670944:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670950:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.670958:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670966:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10060d2a00/0x68a7113811d7f55f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.670979:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.670982:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.670984:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.671679:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.671704:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c45f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355456:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.671802:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074483.671810:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c44380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355328:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.685244:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.685305:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.685340:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.685481:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.685520:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624635776 00000100:00100000:1.0:1769074483.685533:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624635776 00000100:00100000:1.0:1769074483.685597:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 104 00000100:00100000:1.0:1769074483.685618:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10057f7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33936:x1855007624635776:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.685657:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.685741:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.685748:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10057f7050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33936:x1855007624635776:12345-192.168.204.102@tcp:104: Request processed in 143us (535us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.685756:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 104 00000100:00100000:3.0:1769074483.685944:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624635904 00000100:00100000:2.0:1769074483.685948:0:13990:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1769074483.685949:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624635904 00000100:00100000:2.0:1769074483.685964:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636032 00000100:00100000:2.0:1769074483.685969:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636032 00000100:00100000:3.0:1769074483.685978:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 19 00000100:00100000:3.0:1769074483.685988:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33939:x1855007624635904:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074483.685998:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686017:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.686039:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.686039:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074483.686050:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33939:x1855007624635904:12345-192.168.204.102@tcp:104: Request processed in 63us (742us total) trans 0 rc 0/0 00010000:00010000:2.0:1769074483.686057:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006132a00 already unused, calling callback (ffffffffc109c1b0) 00000100:00100000:3.0:1769074483.686062:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 19 00010000:00010000:2.0:1769074483.686067:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074483.686081:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 20 00000100:00100000:3.0:1769074483.686089:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007f35450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33941:x1855007624636032:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074483.686099:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686131:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.686135:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1769074483.686144:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007f35450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33941:x1855007624636032:12345-192.168.204.102@tcp:104: Request processed in 55us (802us total) trans 0 rc 0/0 00010000:00010000:2.0:1769074483.686159:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:3.0:1769074483.686162:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 20 00010000:00010000:2.0:1769074483.686190:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686199:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132a00/0x68a7113811d7f607 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c37a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074483.686210:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00010000:00010000:2.0:1769074483.686223:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.686228:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686235:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006132c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.686239:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074483.686245:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c46300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355584:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.686260:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686269:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.686275:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686282:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132c00/0x68a7113811d7f5b3 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c29a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686295:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.686299:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686306:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060d3800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.686310:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074483.686314:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005c47480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355712:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.686342:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686350:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.686369:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074483.686373:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355840:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.686385:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060d3800/0x68a7113811d7f54a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c182 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.686402:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.686405:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.686408:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.686416:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.686483:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.686625:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.686759:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.686793:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.686829:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.686858:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.686881:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.687194:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636160 00000100:00100000:1.0:1769074483.687202:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636160 00000100:00100000:1.0:1769074483.687245:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636288 00000100:00100000:1.0:1769074483.687249:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636288 00000100:00100000:1.0:1769074483.687258:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636416 00000100:00100000:1.0:1769074483.687260:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636416 00000100:00100000:1.0:1769074483.687269:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636544 00000100:00100000:1.0:1769074483.687271:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636544 00000100:00100000:1.0:1769074483.687279:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636672 00000100:00100000:1.0:1769074483.687281:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636672 00000100:00100000:1.0:1769074483.687288:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636800 00000100:00100000:1.0:1769074483.687290:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636800 00000100:00100000:1.0:1769074483.687302:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 105 00000100:00100000:1.0:1769074483.687309:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:6519:x1855007624636160:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687320:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687397:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.687403:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:6519:x1855007624636160:12345-192.168.204.102@tcp:104: Request processed in 95us (778us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.687410:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 105 00000100:00100000:1.0:1769074483.687426:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 106 00000100:00100000:1.0:1769074483.687432:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6521:x1855007624636288:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687440:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687468:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.687473:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:6521:x1855007624636288:12345-192.168.204.102@tcp:104: Request processed in 42us (716us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.687479:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 106 00000100:00100000:1.0:1769074483.687490:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 107 00000100:00100000:1.0:1769074483.687496:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33942:x1855007624636416:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687504:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687531:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074483.687536:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33942:x1855007624636416:12345-192.168.204.102@tcp:104: Request processed in 41us (743us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.687542:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 107 00000100:00100000:1.0:1769074483.687555:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 108 00000100:00100000:1.0:1769074483.687561:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33943:x1855007624636544:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687569:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687619:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074483.687625:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33943:x1855007624636544:12345-192.168.204.102@tcp:104: Request processed in 65us (796us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.687631:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 108 00000100:00100000:1.0:1769074483.687666:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 109 00000100:00100000:1.0:1769074483.687672:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624636672:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687681:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687703:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:1.0:1769074483.687707:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33934:x1855007624636672:12345-192.168.204.102@tcp:104: Request processed in 36us (850us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.687712:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 109 00000100:00100000:1.0:1769074483.687724:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 110 00000100:00100000:1.0:1769074483.687729:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624636800:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.687748:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687772:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:1.0:1769074483.687779:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624636800:12345-192.168.204.102@tcp:104: Request processed in 49us (896us total) trans 0 rc 0/0 00010000:00010000:2.0:1769074483.687785:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00000100:00100000:1.0:1769074483.687786:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 110 00010000:00010000:2.0:1769074483.687796:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687806:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e1a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.687808:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1769074483.687813:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687814:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687845:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006133400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.687850:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687870:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687877:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687880:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.687887:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.687906:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687913:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687918:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1a00/0x68a7113811d7e5b4 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30597f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687924:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133400/0x68a7113811d7f574 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c1f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687949:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1769074483.687950:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074483.687956:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687957:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.687973:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100601d400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.687976:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100a771c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.687978:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.687981:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688001:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 2/0,0 mode: --/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688008:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.688009:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688015:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 1/0,0 mode: --/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688018:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.688030:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601d400/0x68a7113811d7f5dd lrc: 0/0,0 mode: --/PW res: [0x280000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c30a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688037:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688045:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.688047:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771c00/0x68a7113811d7e72e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059be2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688049:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688068:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.688071:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688078:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.688083:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688089:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688091:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100601c000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.688098:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688100:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.688108:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688117:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169200/0x68a7113811d7f685 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c4ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688127:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.688131:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.688133:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.688136:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.688138:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.688146:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.688156:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356096:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.688156:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601c000/0x68a7113811d7f5c8 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x224:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c2d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.688178:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.688181:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.688183:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.688200:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355968:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.688215:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356480:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.688239:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356608:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.688262:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074483.688271:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356224:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.688275:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074483.688284:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356352:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.688322:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.700327:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.701109:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.701147:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:2.0:1769074483.701167:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c44380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355328:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.701185:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:2.0:1769074483.701189:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c45f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355456:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.701215:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:2.0:1769074483.701239:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624637184 00000100:00100000:1.0:1769074483.701251:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:2.0:1769074483.701259:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624637184 00000100:00100000:1.0:1769074483.701278:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:2.0:1769074483.701310:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624637312 00000100:00100000:2.0:1769074483.701313:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624637312 00000100:00100000:2.0:1769074483.701325:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624636928 00000100:00100000:2.0:1769074483.701327:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624636928 00000100:00100000:2.0:1769074483.701335:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624635392 00000100:00100000:2.0:1769074483.701337:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624635392 00000100:00100000:2.0:1769074483.701345:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624637568 00000100:00100000:2.0:1769074483.701347:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624637568 00000100:00100000:2.0:1769074483.701362:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 21 00000100:00100000:2.0:1769074483.701381:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:14023:x1855007624637184:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.701402:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701496:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074483.701504:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:14023:x1855007624637184:12345-192.168.204.102@tcp:104: Request processed in 134us (356us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.701511:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 21 00000100:00100000:2.0:1769074483.701532:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 22 00000100:00100000:2.0:1769074483.701538:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:33940:x1855007624637312:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.701548:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701578:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1769074483.701583:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:33940:x1855007624637312:12345-192.168.204.102@tcp:104: Request processed in 46us (401us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.701598:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 22 00000100:00100000:2.0:1769074483.701628:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 23 00000100:00100000:2.0:1769074483.701634:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33944:x1855007624636928:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.701669:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701703:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1769074483.701709:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33944:x1855007624636928:12345-192.168.204.102@tcp:104: Request processed in 75us (496us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.701714:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 23 00000100:00100000:2.0:1769074483.701738:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 24 00000100:00100000:2.0:1769074483.701745:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6522:x1855007624635392:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.701754:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701783:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:2.0:1769074483.701789:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6522:x1855007624635392:12345-192.168.204.102@tcp:104: Request processed in 45us (539us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.701795:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 24 00000100:00100000:2.0:1769074483.701808:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 25 00000100:00100000:2.0:1769074483.701816:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:8372:x1855007624637568:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.701825:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.701847:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00010000:00010000:2.0:1769074483.701850:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:2.0:1769074483.701858:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:8372:x1855007624637568:12345-192.168.204.102@tcp:104: Request processed in 43us (582us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.701864:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 25 00000100:00100000:0.0:1769074483.701869:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624637056 00000100:00100000:0.0:1769074483.701875:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624637056 00010000:00010000:2.0:1769074483.701889:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1769074483.701897:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701906:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006133200 already unused, calling callback (ffffffffc109c1b0) 00000100:00100000:0.0:1769074483.701911:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 111 00010000:00010000:2.0:1769074483.701915:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.701922:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10032cdc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624637056:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.701936:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.701969:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1769074483.701975:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10032cdc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624637056:12345-192.168.204.102@tcp:104: Request processed in 57us (877us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.701982:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 111 00010000:00010000:2.0:1769074483.701982:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.701993:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.702016:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702025:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133200/0x68a7113811d7f59e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c262 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702046:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1769074483.702055:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702064:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006168000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.702065:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074483.702068:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702071:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702086:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006133800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.702090:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702104:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702113:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.702116:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702130:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702130:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.702143:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702144:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d7f65b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c45a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702165:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006133800/0x68a7113811d7f61c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c3b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702189:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074483.702192:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.702194:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702197:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702208:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006132600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.702213:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006168800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.702215:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.702220:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702241:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702249:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.702254:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702262:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006132600/0x68a7113811d7f589 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c22a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702277:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.702280:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702287:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100601ca00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.702291:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702308:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702314:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.702320:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702326:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100601ca00/0x68a7113811d7f5f2 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c342 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.702340:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.702342:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.702343:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.702362:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356992:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.702417:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612357248:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.702446:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074483.702451:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356736:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.702470:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612357120:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.702508:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.702914:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612356864:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.703944:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.703963:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.704003:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.704013:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d7f6af lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c53a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.704040:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.704042:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.704044:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.704062:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074483.704083:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357376:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074483.704774:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.705781:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c46300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355584:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.716407:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612355840:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.716785:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.716807:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.716841:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624638080 00000100:00100000:0.0:1769074483.716850:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624638080 00000100:00100000:0.0:1769074483.716906:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624638208 00000100:00100000:0.0:1769074483.716910:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624638208 00000100:00100000:0.0:1769074483.716928:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 112 00000100:00100000:0.0:1769074483.716937:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:32370:x1855007624638080:12345-192.168.204.102@tcp:104: 00000100:00100000:2.0:1769074483.716937:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005c47480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355712:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.716958:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717058:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.717066:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:32370:x1855007624638080:12345-192.168.204.102@tcp:104: Request processed in 129us (283us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.717074:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 112 00000100:00100000:0.0:1769074483.717095:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 113 00000100:00100000:0.0:1769074483.717102:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624638208:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.717105:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.717110:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717112:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717121:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.717128:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717142:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.717146:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33941:x1855007624638208:12345-192.168.204.102@tcp:104: Request processed in 45us (339us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.717152:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 113 00000100:00100000:0.0:1769074483.717189:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00010000:00010000:2.0:1769074483.717196:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717203:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.717223:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717231:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d7f703 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c61a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717254:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.717258:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717265:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006171800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.717268:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717287:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717293:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.717299:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717305:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171800/0x68a7113811d7f6d9 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.717318:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.717320:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.717322:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.717338:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612357504:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.717395:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074483.717402:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612357632:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.717407:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624638464 00000100:00100000:0.0:1769074483.717412:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624638464 00000100:00100000:0.0:1769074483.717428:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 114 00010000:00010000:2.0:1769074483.717434:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.717435:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33936:x1855007624638464:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.717443:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074483.717453:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356352:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.717463:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612355968:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.717470:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074483.717472:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356608:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.717475:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33936:x1855007624638464:12345-192.168.204.102@tcp:104: Request processed in 42us (288us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.717481:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 114 00010000:00010000:0.0:1769074483.717496:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.717501:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717509:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10087f1800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.717512:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717534:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717542:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.717554:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717560:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1800/0x68a7113811d7e743 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.717575:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.717576:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.717578:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.717591:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357760:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.719034:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356224:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.719055:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356096:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.719065:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612356480:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.719686:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356992:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.727827:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.727892:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.727915:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.727939:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.727961:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.728031:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357760:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.728046:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357376:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.728056:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612356864:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.728125:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074483.728144:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624638592 00000100:00100000:0.0:1769074483.728152:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624638592 00000100:00100000:0.0:1769074483.728209:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624639616 00000100:00100000:0.0:1769074483.728213:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624639616 00000100:00100000:0.0:1769074483.728223:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624639744 00000100:00100000:0.0:1769074483.728226:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624639744 00000100:00100000:2.0:1769074483.732596:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612357248:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.732680:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612356736:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.732696:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612357120:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.733048:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624639872 00000100:00100000:0.0:1769074483.733058:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624639872 00000100:00100000:0.0:1769074483.733092:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 115 00000100:00100000:0.0:1769074483.733102:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6519:x1855007624638592:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.733126:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733421:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.733430:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6519:x1855007624638592:12345-192.168.204.102@tcp:104: Request processed in 329us (5620us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.733438:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 115 00000100:00100000:0.0:1769074483.733462:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 116 00000100:00100000:0.0:1769074483.733470:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33944:x1855007624639616:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.733480:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733516:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.733522:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33944:x1855007624639616:12345-192.168.204.102@tcp:104: Request processed in 53us (5630us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.733527:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 116 00000100:00100000:0.0:1769074483.733541:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 117 00000100:00100000:0.0:1769074483.733547:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33943:x1855007624639744:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.733557:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733584:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074483.733589:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33943:x1855007624639744:12345-192.168.204.102@tcp:104: Request processed in 42us (5674us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.733594:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 117 00000100:00100000:0.0:1769074483.733605:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 118 00000100:00100000:0.0:1769074483.733623:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006029c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624639872:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.733631:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733684:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074483.733690:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006029c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624639872:12345-192.168.204.102@tcp:104: Request processed in 68us (5751us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.733695:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 118 00010000:00010000:0.0:1769074483.733733:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074483.733740:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733749:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060e1600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.733755:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733808:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733815:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.733850:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733858:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10060e1600/0x68a7113811d7f646 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c422 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733882:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074483.733885:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733891:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006170a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.733895:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733912:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733919:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.733925:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733931:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170a00/0x68a7113811d7f6c4 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c572 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733949:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074483.733952:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733958:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100601c200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.733961:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733977:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733983:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.733988:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.733994:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100601c200/0x68a7113811d7f781 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c76a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734007:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.734010:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734016:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006171000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.734019:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734035:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734042:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.734047:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734053:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006171000/0x68a7113811d7f757 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.734068:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.734069:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.734071:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.734091:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358016:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.734154:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358272:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.734185:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074483.734192:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357888:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.734214:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358144:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.734521:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624640000 00000100:00100000:3.0:1769074483.734530:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624640000 00000100:00100000:3.0:1769074483.734580:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 26 00000100:00100000:3.0:1769074483.734589:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33934:x1855007624640000:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074483.734599:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734677:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074483.734683:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33934:x1855007624640000:12345-192.168.204.102@tcp:104: Request processed in 97us (6721us total) trans 0 rc 0/0 00000100:00100000:3.0:1769074483.734691:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 26 00010000:00010000:3.0:1769074483.734711:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074483.734715:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734723:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006168c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074483.734726:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734753:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734759:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074483.734772:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734780:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d7f69a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c502 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.734797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.734798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.734800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.734821:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612358400:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.734875:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612357632:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.734888:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612357504:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.734981:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.734989:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.734995:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.735853:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.735962:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.736872:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.736902:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624640128 00000100:00100000:0.0:1769074483.736911:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624640128 00000100:00100000:0.0:1769074483.736955:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624640512 00000100:00100000:0.0:1769074483.736958:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624640512 00000100:00100000:0.0:1769074483.736967:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624640640 00000100:00100000:0.0:1769074483.736969:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624640640 00000100:00100000:0.0:1769074483.736983:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 119 00000100:00100000:0.0:1769074483.736992:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:32370:x1855007624640128:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.737009:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.737085:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.737092:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:32370:x1855007624640128:12345-192.168.204.102@tcp:104: Request processed in 101us (1248us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.737100:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 119 00000100:00100000:0.0:1769074483.737119:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 120 00000100:00100000:0.0:1769074483.737126:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624640512:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.737134:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.737170:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.737176:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624640512:12345-192.168.204.102@tcp:104: Request processed in 51us (1214us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.737181:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 120 00000100:00100000:0.0:1769074483.737195:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 121 00000100:00100000:0.0:1769074483.737201:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624640640:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.737208:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.737474:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074483.737482:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8372:x1855007624640640:12345-192.168.204.102@tcp:104: Request processed in 280us (615us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.737488:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 121 00010000:00010000:2.0:1769074483.747103:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1769074483.747125:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.747141:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.747150:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.747226:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.749897:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.749958:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.749969:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d7f6ee lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c5e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074483.750039:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358528:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.750213:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.750220:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750566:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006171a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.750575:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750694:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750702:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.750716:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750724:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006171a00/0x68a7113811d7f670 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c492 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074483.750750:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074483.750762:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358656:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.750815:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.750820:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750827:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.750832:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750854:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750867:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.750877:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.750886:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d7f718 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c652 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074483.750910:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358784:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.750947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.750949:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.750953:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.753855:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.753972:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.754034:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624640384 00000100:00100000:0.0:1769074483.754046:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624640384 00000100:00100000:0.0:1769074483.754105:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 122 00000100:00100000:0.0:1769074483.754116:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33941:x1855007624640384:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.754141:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.754225:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.754233:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33941:x1855007624640384:12345-192.168.204.102@tcp:104: Request processed in 119us (268us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.754242:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 122 00000100:00100000:0.0:1769074483.754332:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.754348:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624641664 00000100:00100000:0.0:1769074483.754352:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624641664 00000100:00100000:0.0:1769074483.754486:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 123 00000100:00100000:0.0:1769074483.754494:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33936:x1855007624641664:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.754503:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.755471:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.758719:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.758839:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.758900:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.758924:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624641792 00000100:00100000:1.0:1769074483.758933:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624641792 00000100:00100000:1.0:1769074483.758970:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624641920 00000100:00100000:1.0:1769074483.758973:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624641920 00000100:00100000:1.0:1769074483.759067:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 124 00000100:00100000:1.0:1769074483.759079:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6522:x1855007624641792:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.759095:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.759719:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074483.759727:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:6522:x1855007624641792:12345-192.168.204.102@tcp:104: Request processed in 649us (4978us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.759734:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 124 00000100:00100000:1.0:1769074483.759865:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 125 00000100:00100000:1.0:1769074483.759874:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006028450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624641920:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.759884:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.759911:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074483.759916:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006028450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624641920:12345-192.168.204.102@tcp:104: Request processed in 45us (1208us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.759922:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 125 00000100:00100000:1.0:1769074483.759958:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.760070:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.760093:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.760732:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.760876:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.760929:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612357888:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.760942:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358144:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.760951:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358016:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.760959:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612358272:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.761000:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624642176 00000100:00100000:1.0:1769074483.761015:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624642176 00000100:00100000:1.0:1769074483.761040:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624642304 00000100:00100000:1.0:1769074483.761043:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624642304 00000100:00100000:1.0:1769074483.761053:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624642432 00000100:00100000:1.0:1769074483.761055:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624642432 00000100:00100000:1.0:1769074483.761063:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643200 00000100:00100000:1.0:1769074483.761066:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643200 00000100:00100000:1.0:1769074483.761074:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643328 00000100:00100000:1.0:1769074483.761076:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643328 00000100:00100000:1.0:1769074483.761088:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 126 00000100:00100000:1.0:1769074483.761094:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33950:x1855007624642176:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.761736:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1769074483.761756:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761771:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.761780:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761861:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761870:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.761903:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761912:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1006169600/0x68a7113811d7f72d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c68a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761945:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1769074483.761950:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761957:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100602c800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.761961:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761985:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.761991:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.761997:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762004:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100602c800/0x68a7113811d7f76c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c732 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762020:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.762024:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762030:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ac400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.762034:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762052:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762059:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.762064:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762071:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d7e76d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762085:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.762089:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762095:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c69600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.762098:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762117:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762123:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.762127:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762134:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69600/0x68a7113811d7e7eb lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059dda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.762147:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.762148:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.762151:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.762187:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359040:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.762338:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1769074483.762352:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:33936:x1855007624641664:12345-192.168.204.102@tcp:104: Request processed in 7855us (8016us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.762364:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 123 00000100:00100000:0.0:1769074483.762424:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 127 00000100:00100000:0.0:1769074483.762436:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:14023:x1855007624642304:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.762450:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762515:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.762520:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:14023:x1855007624642304:12345-192.168.204.102@tcp:104: Request processed in 89us (2533us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.762527:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 127 00000100:00100000:0.0:1769074483.762545:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 128 00000100:00100000:0.0:1769074483.762553:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:6521:x1855007624642432:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.762563:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762586:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.762591:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:6521:x1855007624642432:12345-192.168.204.102@tcp:104: Request processed in 40us (2498us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.762596:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 128 00000100:00100000:0.0:1769074483.762620:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 129 00000100:00100000:0.0:1769074483.762627:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33942:x1855007624643200:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.762636:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762671:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074483.762676:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33942:x1855007624643200:12345-192.168.204.102@tcp:104: Request processed in 50us (1948us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.762682:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 129 00000100:00100000:0.0:1769074483.762697:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 130 00000100:00100000:0.0:1769074483.762703:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:33934:x1855007624643328:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.762711:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.762712:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762739:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074483.762748:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:33934:x1855007624643328:12345-192.168.204.102@tcp:104: Request processed in 43us (1871us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.762754:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 130 00010000:00010000:0.0:1769074483.762778:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074483.762783:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762791:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006170e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.762795:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762819:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762826:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.762848:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.762856:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1006170e00/0x68a7113811d7f742 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c6c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.762882:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612359424:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.762947:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074483.762962:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33950:x1855007624642176:12345-192.168.204.102@tcp:104: Request processed in 1863us (2999us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.762973:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 126 00000100:00100000:2.0:1769074483.762977:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359296:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.763006:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074483.763012:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358912:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.763027:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359168:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.763045:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1769074483.763049:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763053:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e0600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.763056:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763072:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763077:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.763083:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763088:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0600/0x68a7113811d7e5c9 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305982a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763099:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1769074483.763100:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763105:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1042168200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.763107:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763117:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763120:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.763123:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763127:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168200/0x68a7113811d7e782 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cc2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763137:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.763139:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763143:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10087f1000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.763146:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763155:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763160:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.763164:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763171:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10087f1000/0x68a7113811d7e815 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763184:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.763187:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763193:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100a771e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.763197:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763215:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763222:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.763227:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763233:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771e00/0x68a7113811d7e7c1 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.763445:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.763446:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.763447:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.763458:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359680:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.763479:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359936:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.763494:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074483.763497:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359552:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.763508:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359808:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.763528:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.765280:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.765289:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.765293:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.765464:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612358400:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.766359:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.766573:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.766674:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.766699:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643456 00000100:00100000:1.0:1769074483.766708:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643456 00000100:00100000:1.0:1769074483.766751:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643584 00000100:00100000:1.0:1769074483.766755:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643584 00000100:00100000:1.0:1769074483.766771:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 131 00000100:00100000:1.0:1769074483.766781:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33944:x1855007624643456:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.766799:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.766879:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.766887:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33944:x1855007624643456:12345-192.168.204.102@tcp:104: Request processed in 108us (544us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.766896:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 131 00000100:00100000:1.0:1769074483.766914:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 132 00000100:00100000:1.0:1769074483.766920:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c43050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624643584:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.766928:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.766953:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.766961:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c43050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624643584:12345-192.168.204.102@tcp:104: Request processed in 39us (387us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.766967:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 132 00000100:00100000:1.0:1769074483.767016:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.767033:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643712 00000100:00100000:1.0:1769074483.767037:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643712 00000100:00100000:1.0:1769074483.767058:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 133 00000100:00100000:1.0:1769074483.767063:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624643712:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.767071:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767095:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074483.767099:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624643712:12345-192.168.204.102@tcp:104: Request processed in 37us (84us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.767106:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 133 00010000:00010000:1.0:1769074483.767122:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074483.767133:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767141:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ada00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.767148:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767192:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767200:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.767223:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767230:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ada00/0x68a7113811d7e758 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059c52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.767267:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360064:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.767329:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074483.767333:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767341:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c69a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.767345:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767367:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767384:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.767396:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767403:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c69a00/0x68a7113811d7e83f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059eba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.767427:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612360192:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.767462:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.767466:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.767472:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10057c5e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.767476:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.767783:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.767797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.770057:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.770599:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.770654:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.770681:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.770690:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10057c5e00/0x68a7113811d7f796 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305c7a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.770755:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360320:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.773048:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.773122:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624643968 00000100:00100000:1.0:1769074483.773132:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624643968 00000100:00100000:2.0:1769074483.773141:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358528:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.773217:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 134 00000100:00100000:1.0:1769074483.773230:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624643968:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.773247:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.773321:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.773328:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:32370:x1855007624643968:12345-192.168.204.102@tcp:104: Request processed in 101us (295us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.773335:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 134 00010000:00010000:2.0:1769074483.773362:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.773376:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.773379:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.773382:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.773396:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100a771000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.773398:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.773408:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.773476:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.773484:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.773512:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.773521:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771000/0x68a7113811d7e7ac lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059d32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.773553:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.773556:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.773558:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.773577:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360448:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074483.773717:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.773744:0:13990:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1769074483.773763:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624642048 00000100:00100000:3.0:1769074483.773769:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624642048 00000100:00100000:3.0:1769074483.773810:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 27 00000100:00100000:3.0:1769074483.773818:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10096a8c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33937:x1855007624642048:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074483.773830:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.773868:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074483.773874:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10096a8c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33937:x1855007624642048:12345-192.168.204.102@tcp:104: Request processed in 58us (15123us total) trans 0 rc 0/0 00000100:00100000:3.0:1769074483.773880:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 27 00010000:00010000:2.0:1769074483.773904:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.773915:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.773927:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100a771800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.773933:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.774001:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.774009:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.774044:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.774053:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a771800/0x68a7113811d7e797 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059cfa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074483.774082:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074483.774094:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360576:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.774133:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.774136:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.774139:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.774846:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.774873:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624644224 00000100:00100000:1.0:1769074483.774880:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624644224 00000100:00100000:1.0:1769074483.774912:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 135 00000100:00100000:1.0:1769074483.774921:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33940:x1855007624644224:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.774932:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.774982:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.775017:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33940:x1855007624644224:12345-192.168.204.102@tcp:104: Request processed in 97us (177us total) trans 0 rc 0/0 00010000:00010000:2.0:1769074483.775026:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1769074483.775027:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 135 00010000:00010000:2.0:1769074483.775043:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.775056:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c68600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.775065:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.775111:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.775119:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.775142:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.775152:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68600/0x68a7113811d7e7d6 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059da2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.775174:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.775176:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.775178:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.775198:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360704:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.775903:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358656:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.782160:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358784:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.809852:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612359424:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810016:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359552:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810031:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359808:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810046:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359936:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810058:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612358912:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810072:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359040:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.810086:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359296:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.810428:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360064:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.810630:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360320:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.810750:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360576:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.810840:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810869:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810888:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810911:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810935:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810966:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.810991:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.811012:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.811030:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.811050:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.811070:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.812604:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.812728:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624645376 00000100:00100000:1.0:1769074483.812744:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624645376 00000100:00100000:1.0:1769074483.812812:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624645504 00000100:00100000:1.0:1769074483.812817:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624645504 00000100:00100000:0.0:1769074483.812920:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.813057:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.813153:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612360192:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.813199:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624645632 00000100:00100000:0.0:1769074483.813206:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624645632 00000100:00100000:0.0:1769074483.813230:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624645760 00000100:00100000:0.0:1769074483.813249:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624645760 00000100:00100000:0.0:1769074483.813263:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624647296 00000100:00100000:0.0:1769074483.813266:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624647296 00000100:00100000:0.0:1769074483.813275:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624647808 00000100:00100000:0.0:1769074483.813279:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624647808 00000100:00100000:0.0:1769074483.813299:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 136 00000100:00100000:0.0:1769074483.813308:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33941:x1855007624645376:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.813327:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.813434:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.813442:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33941:x1855007624645376:12345-192.168.204.102@tcp:104: Request processed in 134us (2606us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.813451:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 136 00000100:00100000:0.0:1769074483.813478:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 137 00000100:00100000:0.0:1769074483.813486:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33940:x1855007624645632:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.813495:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.813531:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.813539:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33940:x1855007624645632:12345-192.168.204.102@tcp:104: Request processed in 53us (2649us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.813546:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 137 00000100:00100000:0.0:1769074483.813562:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 138 00000100:00100000:0.0:1769074483.813569:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c41850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33936:x1855007624645760:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.813583:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.813622:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074483.813628:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c41850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33936:x1855007624645760:12345-192.168.204.102@tcp:104: Request processed in 60us (2716us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.813636:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 138 00000100:00100000:0.0:1769074483.813676:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 139 00000100:00100000:0.0:1769074483.813683:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624647296:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.813691:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.813728:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074483.813735:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624647296:12345-192.168.204.102@tcp:104: Request processed in 52us (824us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.813742:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 139 00000100:00100000:0.0:1769074483.813757:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 140 00000100:00100000:0.0:1769074483.813765:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624647808:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.813776:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.813802:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1769074483.813808:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6521:x1855007624647808:12345-192.168.204.102@tcp:104: Request processed in 44us (752us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.813815:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 140 00010000:00010000:0.0:1769074483.814018:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1769074483.814028:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814038:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005618a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.814047:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814114:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814123:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.814149:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814158:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618a00/0x68a7113811d7e893 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f9a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814189:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074483.814194:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814203:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c68e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.814207:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814228:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814235:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.814243:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814253:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c68e00/0x68a7113811d7e869 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814273:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074483.814276:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814282:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005619400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.814285:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814298:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814304:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.814308:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814316:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619400/0x68a7113811d7e911 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814330:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074483.814334:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814341:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005619800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.814345:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814361:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814369:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.814374:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814383:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619800/0x68a7113811d7e93b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a15a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814397:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.814401:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814407:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ac600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.814410:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814432:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814439:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.814446:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814453:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d7e5de lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059862 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.814469:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.814470:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.814474:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.814498:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360960:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.814580:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361216:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.814619:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1769074483.814627:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360832:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.814719:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361088:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.814745:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361344:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.814923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.814934:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.815028:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 141 00000100:00100000:1.0:1769074483.815038:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6522:x1855007624645504:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.815054:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815108:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.815115:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6522:x1855007624645504:12345-192.168.204.102@tcp:104: Request processed in 79us (4246us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.815124:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 141 00010000:00010000:1.0:1769074483.815179:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.815186:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815194:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c68000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.815201:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815265:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815273:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.815293:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815303:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68000/0x68a7113811d7e800 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.815322:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.815324:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.815325:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.815331:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074483.815345:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612361472:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.815525:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359168:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.815579:0:13989:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1769074483.815599:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624645888 00000100:00100000:2.0:1769074483.815617:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624645888 00000100:00100000:2.0:1769074483.815659:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624646144 00000100:00100000:2.0:1769074483.815663:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624646144 00000100:00100000:2.0:1769074483.815674:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624646016 00000100:00100000:2.0:1769074483.815677:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624646016 00000100:00100000:2.0:1769074483.815686:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624646272 00000100:00100000:2.0:1769074483.815689:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624646272 00000100:00100000:2.0:1769074483.815700:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624646912 00000100:00100000:2.0:1769074483.815704:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624646912 00000100:00100000:2.0:1769074483.815717:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624647040 00000100:00100000:2.0:1769074483.815723:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624647040 00000100:00100000:2.0:1769074483.815736:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624647168 00000100:00100000:2.0:1769074483.815740:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624647168 00000100:00100000:2.0:1769074483.815753:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 28 00000100:00100000:2.0:1769074483.815761:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:33944:x1855007624645888:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.815773:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.815822:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074483.815828:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:33944:x1855007624645888:12345-192.168.204.102@tcp:104: Request processed in 69us (4894us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.815835:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 28 00000100:00100000:2.0:1769074483.815853:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 29 00000100:00100000:2.0:1769074483.815860:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:33939:x1855007624646144:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.815871:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.815898:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1769074483.815905:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c1880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:33939:x1855007624646144:12345-192.168.204.102@tcp:104: Request processed in 45us (4939us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.815912:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 29 00000100:00100000:2.0:1769074483.815926:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 30 00000100:00100000:2.0:1769074483.815933:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:33935:x1855007624646016:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.815943:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.815971:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1769074483.815978:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c0000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:33935:x1855007624646016:12345-192.168.204.102@tcp:104: Request processed in 46us (4986us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.815985:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 30 00000100:00100000:2.0:1769074483.815995:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 31 00000100:00100000:2.0:1769074483.816003:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:33938:x1855007624646272:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.816012:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816033:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:2.0:1769074483.816040:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c2300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:33938:x1855007624646272:12345-192.168.204.102@tcp:104: Request processed in 38us (5028us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.816047:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 31 00000100:00100000:2.0:1769074483.816058:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 32 00000100:00100000:2.0:1769074483.816065:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:33934:x1855007624646912:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.816074:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816092:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:2.0:1769074483.816098:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:33934:x1855007624646912:12345-192.168.204.102@tcp:104: Request processed in 34us (5069us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.816105:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 32 00000100:00100000:2.0:1769074483.816116:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 33 00000100:00100000:2.0:1769074483.816123:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33943:x1855007624647040:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.816135:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816156:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:2.0:1769074483.816161:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10056c3480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33943:x1855007624647040:12345-192.168.204.102@tcp:104: Request processed in 39us (5111us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.816167:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 33 00000100:00100000:2.0:1769074483.816181:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 34 00000100:00100000:2.0:1769074483.816189:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007f37c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:32370:x1855007624647168:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074483.816199:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816218:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:2.0:1769074483.816226:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007f37c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:32370:x1855007624647168:12345-192.168.204.102@tcp:104: Request processed in 38us (5155us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074483.816233:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 34 00010000:00010000:2.0:1769074483.816270:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:2.0:1769074483.816275:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816283:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005619000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816289:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816326:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816332:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.816344:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816353:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005619000/0x68a7113811d7e8bd lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a00a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816379:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:2.0:1769074483.816382:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816390:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c68800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816395:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816426:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816435:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.816442:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816451:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68800/0x68a7113811d7e854 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059ef2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816469:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1769074483.816474:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816482:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c69200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816487:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816505:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816512:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.816516:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816524:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69200/0x68a7113811d7e82a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059e82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816545:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1769074483.816549:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816556:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005618000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816561:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816580:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816586:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.816591:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816600:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005618000/0x68a7113811d7e8e7 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a07a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816635:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1769074483.816687:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816695:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c69e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816698:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816716:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816724:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.816729:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816737:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c69e00/0x68a7113811d7e87e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059f62 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816750:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074483.816753:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.816759:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005697800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.816762:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.817719:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.817732:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.817742:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1042168000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.817747:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.817786:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.817793:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.817819:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.817826:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1042168000/0x68a7113811d7e8a8 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059fd2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.817867:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612362240:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.817991:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.817994:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.817995:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.818001:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.818034:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.818167:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.818209:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.818249:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624647936 00000100:00100000:1.0:1769074483.818258:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624647936 00000100:00100000:1.0:1769074483.818303:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624648192 00000100:00100000:1.0:1769074483.818307:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624648192 00000100:00100000:1.0:1769074483.818324:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 142 00000100:00100000:1.0:1769074483.818332:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624647936:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.818343:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818391:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.818414:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624647936:12345-192.168.204.102@tcp:104: Request processed in 83us (247us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.818421:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 142 00000100:00100000:1.0:1769074483.818441:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 143 00000100:00100000:1.0:1769074483.818447:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33950:x1855007624648192:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.818456:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818487:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.818492:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33950:x1855007624648192:12345-192.168.204.102@tcp:104: Request processed in 46us (283us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.818498:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 143 00010000:00010000:1.0:1769074483.818539:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074483.818544:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818551:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e0a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.818556:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818580:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818587:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.818600:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818618:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e0a00/0x68a7113811d7e8fc lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a0b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.818680:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612362368:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.818747:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.818751:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818758:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1042168600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.818762:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818786:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818792:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.818802:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.818808:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1042168600/0x68a7113811d7e98f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a23a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074483.818831:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074483.818839:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612362496:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074483.818889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.818891:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.818892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.819032:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.819041:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.819048:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.819055:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005697800/0x68a7113811d7e965 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a1ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.819071:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.819073:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.819076:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.819114:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612359680:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819138:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361600:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819186:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361856:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819211:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612362112:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819231:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1769074483.819346:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361728:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819377:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361984:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.819395:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612362624:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.820949:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.833454:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360448:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.834154:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.834337:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624648320 00000100:00100000:0.0:1769074483.834353:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624648320 00000100:00100000:0.0:1769074483.834444:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 144 00000100:00100000:0.0:1769074483.834462:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624648320:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.834489:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834614:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.834622:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624648320:12345-192.168.204.102@tcp:104: Request processed in 167us (1982us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.834630:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 144 00010000:00010000:0.0:1769074483.834696:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.834707:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834719:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005619600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.834730:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834808:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834825:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.834858:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834867:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005619600/0x68a7113811d7e8d2 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a042 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.834903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.834906:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.834908:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.834929:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074483.834971:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612362752:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.835349:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612361472:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.835462:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612362240:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.835906:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612360704:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.836162:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10056c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360832:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.836201:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361088:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.836214:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361344:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.836226:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612360960:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.836252:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612361216:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.843287:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612362496:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.843314:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612362368:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.843457:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361728:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.845055:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612362752:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.850943:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.851531:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624649088 00000100:00100000:1.0:1769074483.851554:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624649088 00000100:00100000:1.0:1769074483.851699:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 145 00000100:00100000:1.0:1769074483.851712:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33943:x1855007624649088:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.851738:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.851873:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.851880:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33943:x1855007624649088:12345-192.168.204.102@tcp:104: Request processed in 172us (948us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.851888:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 145 00010000:00010000:3.0:1769074483.852006:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074483.852019:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.852032:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c3a200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074483.852040:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.852110:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.852122:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1769074483.852124:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00010000:00010000:3.0:1769074483.852158:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.852169:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624648960 00000100:00100000:1.0:1769074483.852174:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00010000:00010000:3.0:1769074483.852178:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a200/0x68a7113811d7ea0d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a38a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074483.852179:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624648960 00010000:00010000:3.0:1769074483.852216:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.852219:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.852221:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.852246:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074483.852256:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 146 00000100:00100000:3.0:1769074483.852263:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612362880:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.852265:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c42c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33934:x1855007624648960:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.852275:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.852401:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.852409:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c42c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33934:x1855007624648960:12345-192.168.204.102@tcp:104: Request processed in 145us (289us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.852420:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 146 00000100:00100000:2.0:1769074483.852579:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361984:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.852597:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.852616:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.852626:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005caca00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.852631:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.852687:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.852697:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.852709:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.852716:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005caca00/0x68a7113811d7e5f3 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305989a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.852738:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.852739:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.852752:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.852800:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612362624:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.852817:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361856:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.852827:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612362112:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.852850:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074483.852863:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612363008:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.858931:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.859153:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624649600 00000100:00100000:1.0:1769074483.859195:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624649600 00000100:00100000:1.0:1769074483.859255:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 147 00000100:00100000:1.0:1769074483.859265:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6522:x1855007624649600:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.859292:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.859366:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.859373:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6522:x1855007624649600:12345-192.168.204.102@tcp:104: Request processed in 110us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.859381:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 147 00010000:00010000:0.0:1769074483.869258:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.869273:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869296:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1000031c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.869304:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869370:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869379:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.869407:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869415:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031c00/0x68a7113811d7ea37 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3fa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869437:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.869439:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.869441:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.869461:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363136:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.869600:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.869666:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624650112 00000100:00100000:0.0:1769074483.869674:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624650112 00000100:00100000:0.0:1769074483.869721:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 148 00000100:00100000:0.0:1769074483.869731:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624650112:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.869743:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869791:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.869798:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624650112:12345-192.168.204.102@tcp:104: Request processed in 68us (199us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.869806:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 148 00010000:00010000:0.0:1769074483.869828:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.869833:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869841:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1000030600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.869846:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869866:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869873:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.869885:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869894:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030600/0x68a7113811d7e9b9 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.869908:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.869909:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.869911:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.869920:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074483.869928:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363264:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.870013:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.870029:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624650240 00000100:00100000:0.0:1769074483.870034:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624650240 00000100:00100000:0.0:1769074483.870057:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 149 00000100:00100000:0.0:1769074483.870063:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624650240:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.870073:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870105:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.870110:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624650240:12345-192.168.204.102@tcp:104: Request processed in 48us (99us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.870116:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 149 00010000:00010000:0.0:1769074483.870132:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.870137:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870144:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005618200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.870148:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870170:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870177:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.870187:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870201:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618200/0x68a7113811d7e926 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a122 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870216:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.870218:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.870220:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.870229:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363392:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.870294:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.870309:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624650368 00000100:00100000:0.0:1769074483.870313:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624650368 00000100:00100000:0.0:1769074483.870333:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 150 00000100:00100000:0.0:1769074483.870339:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624650368:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.870350:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870388:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.870394:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33939:x1855007624650368:12345-192.168.204.102@tcp:104: Request processed in 56us (101us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.870401:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 150 00000100:00100000:0.0:1769074483.870443:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.870455:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624650752 00000100:00100000:0.0:1769074483.870459:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624650752 00000100:00100000:0.0:1769074483.870478:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 151 00000100:00100000:0.0:1769074483.870484:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33938:x1855007624650752:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.870492:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870521:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.870526:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33938:x1855007624650752:12345-192.168.204.102@tcp:104: Request processed in 43us (84us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.870533:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 151 00010000:00010000:0.0:1769074483.870547:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074483.870551:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870557:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e1800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.870561:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870583:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870588:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.870598:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870604:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e1800/0x68a7113811d7e9e3 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a31a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870678:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.870681:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870687:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c68200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.870691:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870710:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870716:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.870722:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870729:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c68200/0x68a7113811d7e950 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a192 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870742:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.870744:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.870746:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.870758:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363648:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.870789:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074483.870795:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363520:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.870847:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.870862:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624650880 00000100:00100000:0.0:1769074483.870867:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624650880 00000100:00100000:0.0:1769074483.870894:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 152 00000100:00100000:0.0:1769074483.870901:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624650880:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.870909:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870936:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.870942:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33937:x1855007624650880:12345-192.168.204.102@tcp:104: Request processed in 42us (96us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.870949:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 152 00010000:00010000:0.0:1769074483.870962:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074483.870966:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870974:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005697e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074483.870978:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.870998:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.871005:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074483.871015:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.871022:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697e00/0x68a7113811d7e9a4 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a272 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.871035:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.871037:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.871039:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.871047:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074483.871054:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363776:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.871122:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00010000:00010000:2.0:1769074483.872787:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.872895:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612361600:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.873425:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074483.873449:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624651008 00000100:00100000:0.0:1769074483.873457:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624651008 00000100:00100000:0.0:1769074483.873503:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 153 00000100:00100000:0.0:1769074483.873510:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c40450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624651008:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.873524:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.873586:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00010000:00010000:2.0:1769074483.874424:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.874434:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.874445:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10087f1200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.874451:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.874505:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.874512:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.874535:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.874543:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f1200/0x68a7113811d7e97a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a202 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.874562:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.874563:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.874565:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.874578:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074483.874595:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612363904:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.874730:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c40450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6519:x1855007624651008:12345-192.168.204.102@tcp:104: Request processed in 1219us (3607us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.874738:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 153 00000100:00100000:0.0:1769074483.876031:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.876073:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624651136 00000100:00100000:0.0:1769074483.876084:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624651136 00000100:00100000:0.0:1769074483.876137:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 154 00000100:00100000:0.0:1769074483.876146:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624651136:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.876174:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.877217:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.877237:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624651136:12345-192.168.204.102@tcp:104: Request processed in 1082us (1209us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.877245:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 154 00010000:00010000:2.0:1769074483.877421:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.877429:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.877439:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029aca00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.877445:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.877492:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.877499:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.877519:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.877526:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d7e9ce lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a2e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.877551:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.877553:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.877554:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.877571:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364032:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.884083:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.884137:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.884178:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624651264 00000100:00100000:1.0:1769074483.884194:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624651264 00000100:00100000:1.0:1769074483.884245:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624651392 00000100:00100000:1.0:1769074483.884249:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624651392 00000100:00100000:1.0:1769074483.884266:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 155 00000100:00100000:1.0:1769074483.884275:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:32370:x1855007624651264:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.884293:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.884376:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.884383:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:32370:x1855007624651264:12345-192.168.204.102@tcp:104: Request processed in 109us (310us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.884391:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 155 00000100:00100000:1.0:1769074483.884410:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 156 00000100:00100000:1.0:1769074483.884417:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33944:x1855007624651392:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.884425:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.884512:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.884519:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33944:x1855007624651392:12345-192.168.204.102@tcp:104: Request processed in 103us (382us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.884527:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 156 00010000:00010000:3.0:1769074483.887542:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1769074483.887558:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.887572:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1000031200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074483.887579:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.887986:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.887997:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.888027:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888035:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000031200/0x68a7113811d7ea61 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a46a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888059:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.888064:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888071:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005696200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.888075:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888095:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888102:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.888107:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888127:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696200/0x68a7113811d7ea8b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4da expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.888140:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.888142:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.888145:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.888195:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364288:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.888291:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074483.888298:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364160:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.891429:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.893713:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612362880:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.895889:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.895929:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624651904 00000100:00100000:1.0:1769074483.895938:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624651904 00000100:00100000:1.0:1769074483.895986:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 157 00000100:00100000:1.0:1769074483.895996:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33943:x1855007624651904:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.896011:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.896088:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.896095:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33943:x1855007624651904:12345-192.168.204.102@tcp:104: Request processed in 101us (218us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.896102:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 157 00010000:00010000:2.0:1769074483.896424:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.896432:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.896443:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c3ac00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.896449:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.896508:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.896516:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.896539:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.896547:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3ac00/0x68a7113811d7e9f8 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a352 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.896571:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.896573:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.896575:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.896587:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074483.896602:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364416:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.898450:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.898497:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624652032 00000100:00100000:1.0:1769074483.898506:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624652032 00000100:00100000:1.0:1769074483.898556:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 158 00000100:00100000:1.0:1769074483.898571:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624652032:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.898587:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.898691:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.898699:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8372:x1855007624652032:12345-192.168.204.102@tcp:104: Request processed in 129us (258us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.898707:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 158 00010000:00010000:2.0:1769074483.899039:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.899046:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.899056:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005696600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.899062:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.899108:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.899115:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.899144:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.899151:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696600/0x68a7113811d7ea22 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a3c2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.899172:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.899175:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.899177:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.899200:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074483.899216:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612364544:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.908743:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.908780:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.908812:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.908851:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.908868:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624652160 00000100:00100000:1.0:1769074483.908877:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624652160 00000100:00100000:1.0:1769074483.908919:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624652544 00000100:00100000:1.0:1769074483.908923:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624652544 00000100:00100000:1.0:1769074483.908940:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 159 00000100:00100000:1.0:1769074483.908950:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624652160:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.908968:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909031:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074483.909038:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624652160:12345-192.168.204.102@tcp:104: Request processed in 89us (305us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.909046:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 159 00000100:00100000:1.0:1769074483.909065:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 160 00000100:00100000:1.0:1769074483.909071:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b100602bc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33940:x1855007624652544:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.909089:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909122:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.909128:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b100602bc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33940:x1855007624652544:12345-192.168.204.102@tcp:104: Request processed in 57us (347us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.909135:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 160 00010000:00010000:1.0:1769074483.909182:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074483.909418:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909430:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1000030200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.909436:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909560:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909570:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.909589:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909599:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030200/0x68a7113811d7ea4c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a432 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909633:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.909667:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909688:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005696400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.909693:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909716:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909724:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.909731:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909739:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005696400/0x68a7113811d7eab5 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a54a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.909755:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.909757:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.909759:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.909766:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.909782:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364800:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.909838:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074483.909845:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364672:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.916849:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624652672 00000100:00100000:3.0:1769074483.916866:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624652672 00000100:00100000:3.0:1769074483.916989:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 35 00000100:00100000:3.0:1769074483.917000:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6521:x1855007624652672:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074483.917017:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917090:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074483.917097:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:6521:x1855007624652672:12345-192.168.204.102@tcp:104: Request processed in 100us (8285us total) trans 0 rc 0/0 00000100:00100000:3.0:1769074483.917106:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 35 00010000:00010000:3.0:1769074483.917155:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074483.917174:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917183:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005dece00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074483.917197:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917250:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917257:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074483.917276:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917284:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dece00/0x68a7113811d7eadf lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5ba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074483.917303:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.917305:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074483.917306:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.917318:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769074483.917333:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364928:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.922102:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363136:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.922523:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363264:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.942593:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942664:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942697:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942718:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942734:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942765:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.942791:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.943028:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364672:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.943042:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364800:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.943051:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364160:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.943059:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612364288:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.943126:0:30958:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074483.943141:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624653184 00000100:00100000:1.0:1769074483.943147:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624653184 00000100:00100000:1.0:1769074483.943180:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624653056 00000100:00100000:1.0:1769074483.943183:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624653056 00000100:00100000:0.0:1769074483.943580:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363776:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.943593:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363520:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.943617:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363648:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.943629:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612363392:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.943675:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624654080 00000100:00100000:0.0:1769074483.943680:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624654080 00000100:00100000:0.0:1769074483.943698:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624653952 00000100:00100000:0.0:1769074483.943701:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624653952 00000100:00100000:0.0:1769074483.943712:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624654208 00000100:00100000:0.0:1769074483.943714:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624654208 00000100:00100000:0.0:1769074483.943723:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624654336 00000100:00100000:0.0:1769074483.943726:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624654336 00000100:00100000:0.0:1769074483.943733:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624654464 00000100:00100000:0.0:1769074483.943736:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624654464 00000100:00100000:0.0:1769074483.943749:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 161 00000100:00100000:0.0:1769074483.943757:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:32370:x1855007624653184:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.943775:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.943848:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.943855:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:32370:x1855007624653184:12345-192.168.204.102@tcp:104: Request processed in 99us (1273us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.943862:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 161 00000100:00100000:0.0:1769074483.943880:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 162 00000100:00100000:0.0:1769074483.943885:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33942:x1855007624654080:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.943892:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.943927:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074483.943932:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33942:x1855007624654080:12345-192.168.204.102@tcp:104: Request processed in 47us (1235us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.943938:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 162 00000100:00100000:0.0:1769074483.943951:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 163 00000100:00100000:0.0:1769074483.943958:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33937:x1855007624653952:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.943966:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.943994:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074483.944000:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:33937:x1855007624653952:12345-192.168.204.102@tcp:104: Request processed in 43us (1282us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.944005:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 163 00000100:00100000:0.0:1769074483.944017:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 164 00000100:00100000:0.0:1769074483.944022:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33943:x1855007624654208:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.944030:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.944062:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074483.944067:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33943:x1855007624654208:12345-192.168.204.102@tcp:104: Request processed in 44us (1333us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.944072:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 164 00000100:00100000:0.0:1769074483.944083:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 165 00000100:00100000:0.0:1769074483.944089:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624654336:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.944097:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.944118:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1769074483.944123:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:33940:x1855007624654336:12345-192.168.204.102@tcp:104: Request processed in 35us (1359us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.944129:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 165 00000100:00100000:0.0:1769074483.944140:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 166 00000100:00100000:0.0:1769074483.944146:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c40050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33944:x1855007624654464:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.944152:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.944183:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:1.0:1769074483.945023:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 167 00000100:00100000:1.0:1769074483.945031:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624653056:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.945041:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.945076:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:1.0:1769074483.945080:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624653056:12345-192.168.204.102@tcp:104: Request processed in 51us (2418us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.945087:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 167 00000100:00100000:0.0:1769074483.945348:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c40050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33944:x1855007624654464:12345-192.168.204.102@tcp:104: Request processed in 1200us (2554us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.945355:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 166 00010000:00010000:1.0:1769074483.950293:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:1.0:1769074483.950309:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950322:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c3a000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950346:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950401:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950409:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950440:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950449:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3a000/0x68a7113811d7eb33 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a69a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950473:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:1.0:1769074483.950478:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950484:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005cad000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950488:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950507:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950514:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950520:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950528:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cad000/0x68a7113811d7e61d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305990a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950549:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:1.0:1769074483.950553:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950560:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10097e1e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950564:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950584:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950590:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950596:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950614:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10097e1e00/0x68a7113811d7e608 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf30598d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950629:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1769074483.950633:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950669:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005618400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950673:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950692:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950704:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950710:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950717:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005618400/0x68a7113811d7eaca lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a582 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950731:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074483.950735:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950741:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005697400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950745:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950764:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950771:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950775:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950782:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005697400/0x68a7113811d7ea76 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a4a2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950794:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074483.950797:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950813:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005696000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950817:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950837:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950843:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950848:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950854:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005696000/0x68a7113811d7eaa0 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a512 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950867:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.950870:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950876:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c3ba00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.950879:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950895:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950901:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.950905:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950918:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005c3ba00/0x68a7113811d7eb09 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a62a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.950931:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.950934:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.950936:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.950944:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.951172:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365184:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951257:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365440:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951281:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365696:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951304:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 4 async RPCs [1->0] 00000100:00100000:1.0:1769074483.951311:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365056:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951331:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365312:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951350:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365568:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.951370:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365824:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074483.951711:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074483.951718:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074483.952337:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364416:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.952352:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364032:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.952364:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612363904:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.952396:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612364544:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074483.952407:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612363008:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.953467:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.953972:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074483.954006:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624654976 00000100:00100000:1.0:1769074483.954015:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624654976 00000100:00100000:1.0:1769074483.954057:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624655104 00000100:00100000:1.0:1769074483.954060:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624655104 00000100:00100000:1.0:1769074483.954075:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 168 00000100:00100000:1.0:1769074483.954097:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6522:x1855007624654976:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.954112:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.954177:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074483.954428:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612364928:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074483.956735:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6522:x1855007624654976:12345-192.168.204.102@tcp:104: Request processed in 90us (724us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.956752:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 168 00000100:00100000:1.0:1769074483.956789:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 169 00000100:00100000:1.0:1769074483.956803:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624655104:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074483.956819:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.956891:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074483.956898:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33935:x1855007624655104:12345-192.168.204.102@tcp:104: Request processed in 101us (2928us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074483.956904:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 169 00010000:00010000:1.0:1769074483.956925:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074483.956932:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.956941:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005c3b800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.956947:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.956997:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957010:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.957032:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957039:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005c3b800/0x68a7113811d7eb1e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a662 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957058:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074483.957062:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957068:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005dec600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074483.957071:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957089:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957095:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074483.957102:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957108:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec600/0x68a7113811d7eb48 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a6d2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074483.957121:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.957123:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074483.957125:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074483.957152:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365952:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.957265:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612366080:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.959700:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074483.959735:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624655232 00000100:00100000:0.0:1769074483.959745:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624655232 00000100:00100000:0.0:1769074483.959784:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 170 00000100:00100000:0.0:1769074483.959792:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006027800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624655232:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074483.959807:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074483.959858:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074483.959865:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006027800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624655232:12345-192.168.204.102@tcp:104: Request processed in 73us (174us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074483.959871:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 170 00010000:00010000:2.0:1769074483.960443:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074483.960453:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.960463:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005687200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074483.960469:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.960517:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.960525:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074483.960548:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.960556:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005687200/0x68a7113811d7eaf4 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a5f2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074483.960577:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.960579:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074483.960580:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074483.960595:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612366208:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074483.964712:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074483.971885:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365056:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971907:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365312:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971917:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365568:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971926:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365824:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971937:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365184:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971952:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365440:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.971962:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365696:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.974597:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612366080:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074483.974682:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612365952:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074483.974865:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612366208:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074484.770001:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074485.792715:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074486.818889:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074487.846865:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074487.969152:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612366336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.969257:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.969292:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074487.969299:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.969327:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.976367:0:2364:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477397 gen 1 00000100:00100000:0.0:1769074487.976785:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.976807:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.976818:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612366336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074487.976829:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612366592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074488.763393:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.763433:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.763466:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.763495:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624657024 00000100:00100000:1.0:1769074488.763506:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624657024 00000100:00100000:1.0:1769074488.763558:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624657152 00000100:00100000:1.0:1769074488.763561:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624657152 00000100:00100000:1.0:1769074488.763570:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624657280 00000100:00100000:1.0:1769074488.763572:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624657280 00000100:00100000:1.0:1769074488.763586:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 171 00000100:00100000:1.0:1769074488.763594:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6522:x1855007624657024:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.763608:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.763717:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074488.763724:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6522:x1855007624657024:12345-192.168.204.102@tcp:104: Request processed in 130us (344us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.763731:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 171 00000100:00100000:1.0:1769074488.763748:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 172 00000100:00100000:1.0:1769074488.763754:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624657152:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.763762:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.763804:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074488.763810:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33950:x1855007624657152:12345-192.168.204.102@tcp:104: Request processed in 56us (376us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.763817:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 172 00000100:00100000:1.0:1769074488.763828:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 173 00000100:00100000:1.0:1769074488.763835:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33938:x1855007624657280:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.763844:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.763871:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074488.763876:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33938:x1855007624657280:12345-192.168.204.102@tcp:104: Request processed in 42us (411us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.763882:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 173 00010000:00010000:3.0:1769074488.764059:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1769074488.764067:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764075:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005dec000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074488.764141:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764186:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764194:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074488.764217:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764224:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec000/0x68a7113811d7eb72 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a742 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764246:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1769074488.764249:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764256:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074488.764259:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764277:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764283:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074488.764290:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764297:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686200/0x68a7113811d7ebf0 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a892 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764309:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074488.764313:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764319:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005ded600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074488.764323:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764343:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764349:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074488.764354:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764361:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded600/0x68a7113811d7eb5d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a70a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.764374:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.764376:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.764377:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.764384:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074488.764398:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612366848:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074488.764465:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612367104:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074488.764493:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074488.764499:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612366976:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074488.764598:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074488.770015:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.770056:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.770099:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074488.770117:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658048 00000100:00100000:1.0:1769074488.770127:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658048 00000100:00100000:1.0:1769074488.770177:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 174 00000100:00100000:1.0:1769074488.770186:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c42850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624658048:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.770201:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.770267:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074488.770274:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c42850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33942:x1855007624658048:12345-192.168.204.102@tcp:104: Request processed in 89us (271us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.770281:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 174 00010000:00010000:2.0:1769074488.777649:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074488.777664:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.777677:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005dec400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.777685:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.777747:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.777754:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.777786:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.777793:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005dec400/0x68a7113811d7eb9c lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7b2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.777822:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.777824:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.777826:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074488.777849:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612367232:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074488.780710:0:13989:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658176 00000100:00100000:3.0:1769074488.780726:0:13989:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658176 00000100:00100000:3.0:1769074488.780919:0:13989:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 36 00000100:00100000:3.0:1769074488.780929:0:13989:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33937:x1855007624658176:12345-192.168.204.102@tcp:104: 00010000:00010000:3.0:1769074488.780945:0:13989:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781014:0:13989:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1769074488.781020:0:13989:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:33937:x1855007624658176:12345-192.168.204.102@tcp:104: Request processed in 94us (10964us total) trans 0 rc 0/0 00000100:00100000:3.0:1769074488.781027:0:13989:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 36 00010000:00010000:3.0:1769074488.781051:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074488.781057:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781066:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074488.781072:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781114:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781121:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074488.781138:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781146:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686600/0x68a7113811d7ec6e lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9e2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.781166:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.781167:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.781169:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074488.783660:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612367360:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.784698:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784783:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784815:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784903:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784919:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784933:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784948:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784963:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.784995:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.785014:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.785030:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.785094:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074488.785109:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658432 00000100:00100000:0.0:1769074488.785118:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658432 00000100:00100000:0.0:1769074488.785152:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658560 00000100:00100000:0.0:1769074488.785154:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658560 00000100:00100000:0.0:1769074488.785164:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658688 00000100:00100000:0.0:1769074488.785166:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658688 00000100:00100000:0.0:1769074488.785174:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658816 00000100:00100000:0.0:1769074488.785176:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658816 00000100:00100000:0.0:1769074488.785187:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624659712 00000100:00100000:0.0:1769074488.785189:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624659712 00000100:00100000:0.0:1769074488.785197:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624659840 00000100:00100000:0.0:1769074488.785199:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624659840 00000100:00100000:0.0:1769074488.785207:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624659968 00000100:00100000:0.0:1769074488.785210:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624659968 00000100:00100000:0.0:1769074488.785223:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 175 00000100:00100000:0.0:1769074488.785230:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33936:x1855007624658432:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785246:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785300:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074488.785307:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33936:x1855007624658432:12345-192.168.204.102@tcp:104: Request processed in 77us (614us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785314:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 175 00000100:00100000:0.0:1769074488.785331:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 176 00000100:00100000:0.0:1769074488.785338:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6519:x1855007624658560:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785346:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785377:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074488.785383:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6519:x1855007624658560:12345-192.168.204.102@tcp:104: Request processed in 45us (600us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785389:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 176 00000100:00100000:0.0:1769074488.785400:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 177 00000100:00100000:0.0:1769074488.785405:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006027100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33941:x1855007624658688:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785412:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785440:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074488.785446:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006027100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33941:x1855007624658688:12345-192.168.204.102@tcp:104: Request processed in 41us (631us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785452:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 177 00000100:00100000:0.0:1769074488.785464:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 178 00000100:00100000:0.0:1769074488.785469:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:33935:x1855007624658816:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785477:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785507:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074488.785513:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:33935:x1855007624658816:12345-192.168.204.102@tcp:104: Request processed in 44us (610us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785519:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 178 00000100:00100000:0.0:1769074488.785545:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 179 00000100:00100000:0.0:1769074488.785551:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:32370:x1855007624659712:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785558:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785580:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:0.0:1769074488.785585:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:32370:x1855007624659712:12345-192.168.204.102@tcp:104: Request processed in 35us (666us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785590:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 179 00000100:00100000:0.0:1769074488.785600:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 180 00000100:00100000:0.0:1769074488.785605:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33944:x1855007624659840:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785611:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785632:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:0.0:1769074488.785661:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:33944:x1855007624659840:12345-192.168.204.102@tcp:104: Request processed in 32us (703us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785666:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 180 00000100:00100000:0.0:1769074488.785676:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 181 00000100:00100000:0.0:1769074488.785682:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c42450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624659968:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.785689:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785709:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:0.0:1769074488.785713:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c42450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:14023:x1855007624659968:12345-192.168.204.102@tcp:104: Request processed in 32us (766us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.785719:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 181 00010000:00010000:0.0:1769074488.785750:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:0.0:1769074488.785755:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785761:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1000031000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.785767:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785806:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785812:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.785831:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785837:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000031000/0x68a7113811d7ec1a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a902 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785856:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:0.0:1769074488.785859:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785865:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.785868:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785883:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785889:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.785895:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785901:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d7ebc6 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a822 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785913:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:0.0:1769074488.785916:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785922:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.785925:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785941:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785948:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.785952:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785958:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686a00/0x68a7113811d7ec44 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a972 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785971:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074488.785975:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.785981:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005dec200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.785984:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786003:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786009:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.786013:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786020:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005dec200/0x68a7113811d7eb87 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a77a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786033:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074488.786036:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786042:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.786045:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786060:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786067:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.786072:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786078:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d7ebdb lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a85a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786089:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074488.786093:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786098:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.786101:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786115:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786121:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.786125:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786131:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686800/0x68a7113811d7ec83 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa1a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786142:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074488.786145:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786150:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005ded800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.786152:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786166:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786172:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.786176:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786184:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005ded800/0x68a7113811d7ebb1 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a7ea expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.786195:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.786197:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.786199:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.786206:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.786211:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.786216:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074488.786229:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367488:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786272:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367744:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786289:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368000:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786305:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368256:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786325:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1769074488.786331:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367616:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786346:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367872:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.786360:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368128:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.787829:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612366976:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.787850:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612367104:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.788587:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624660096 00000100:00100000:2.0:1769074488.788597:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624660096 00000100:00100000:2.0:1769074488.788636:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624660224 00000100:00100000:2.0:1769074488.789075:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624660224 00000100:00100000:2.0:1769074488.789095:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624660352 00000100:00100000:2.0:1769074488.789098:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624660352 00000100:00100000:2.0:1769074488.789108:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624658304 00000100:00100000:2.0:1769074488.789111:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624658304 00000100:00100000:2.0:1769074488.789128:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 37 00000100:00100000:2.0:1769074488.789136:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:33943:x1855007624660096:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074488.789150:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.792579:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074488.792601:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:33943:x1855007624660096:12345-192.168.204.102@tcp:104: Request processed in 3458us (7629us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074488.792613:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 37 00000100:00100000:2.0:1769074488.792672:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 38 00000100:00100000:2.0:1769074488.792685:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6521:x1855007624660224:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074488.792701:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796654:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074488.796670:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796684:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005687e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.796692:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796762:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796769:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.796798:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796806:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005687e00/0x68a7113811d7ec59 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a9aa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.796832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.796834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.796835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074488.796856:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612368384:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074488.801365:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074488.801385:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:6521:x1855007624660224:12345-192.168.204.102@tcp:104: Request processed in 8698us (16383us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074488.801404:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 38 00000100:00100000:2.0:1769074488.801440:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 39 00000100:00100000:2.0:1769074488.801454:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33939:x1855007624660352:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074488.801472:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801559:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1769074488.801565:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33939:x1855007624660352:12345-192.168.204.102@tcp:104: Request processed in 118us (16550us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074488.801571:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 39 00000100:00100000:2.0:1769074488.801588:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 40 00000100:00100000:2.0:1769074488.801596:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33940:x1855007624658304:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074488.801605:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801634:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1769074488.801661:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33940:x1855007624658304:12345-192.168.204.102@tcp:104: Request processed in 66us (16630us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074488.801667:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 40 00010000:00010000:2.0:1769074488.801685:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1769074488.801693:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801702:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100164a400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.801709:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801755:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801763:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.801784:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801793:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a400/0x68a7113811d7ec2f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a93a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801812:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074488.801815:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801821:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.801824:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801841:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801846:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.801852:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801858:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d7ec05 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305a8ca expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801869:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074488.801872:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801878:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005cac600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.801880:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801897:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801903:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.801908:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801916:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005cac600/0x68a7113811d7e632 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf3059942 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.801929:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.801930:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.801932:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.801938:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074488.801955:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368512:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.802010:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368768:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.802042:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074488.802049:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368640:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074488.805816:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074488.811266:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.811310:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.811340:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.811599:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624660736 00000100:00100000:0.0:1769074488.811609:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624660736 00000100:00100000:0.0:1769074488.811677:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624660992 00000100:00100000:0.0:1769074488.811681:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624660992 00000100:00100000:0.0:1769074488.811692:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624661120 00000100:00100000:0.0:1769074488.811694:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624661120 00000100:00100000:0.0:1769074488.811708:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 182 00000100:00100000:0.0:1769074488.811716:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33934:x1855007624660736:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.811733:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.811801:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074488.811807:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:33934:x1855007624660736:12345-192.168.204.102@tcp:104: Request processed in 92us (552us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.811814:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 182 00000100:00100000:0.0:1769074488.811832:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 183 00000100:00100000:0.0:1769074488.811838:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6522:x1855007624660992:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.811846:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.811875:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074488.811880:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:6522:x1855007624660992:12345-192.168.204.102@tcp:104: Request processed in 43us (571us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.811886:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 183 00000100:00100000:0.0:1769074488.811897:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 184 00000100:00100000:0.0:1769074488.811902:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8372:x1855007624661120:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.811909:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.811936:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074488.811942:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006137b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8372:x1855007624661120:12345-192.168.204.102@tcp:104: Request processed in 39us (602us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.811947:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 184 00000100:00100000:0.0:1769074488.811993:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367616:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812006:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367872:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812016:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367744:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812024:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368000:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812032:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368256:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074488.812059:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074488.812064:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812072:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.812078:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812122:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812128:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.812145:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812151:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005686c00/0x68a7113811d7ecad lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa8a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812171:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074488.812174:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812180:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005686e00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.812184:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812202:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812208:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.812213:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812219:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005686e00/0x68a7113811d7ec98 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aa52 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812236:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074488.812239:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812245:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.812248:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812263:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812269:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.812273:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812279:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d7ecc2 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aac2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.812292:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.812293:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.812295:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.812301:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.812307:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074488.812320:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369024:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812367:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074488.812373:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612368896:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.812397:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369152:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.821326:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.821370:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.821394:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.821420:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.821481:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.821540:0:13988:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074488.821559:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624661760 00000100:00100000:1.0:1769074488.821569:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624661760 00000100:00100000:1.0:1769074488.821610:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624661888 00000100:00100000:1.0:1769074488.821613:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624661888 00000100:00100000:1.0:1769074488.821625:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624662144 00000100:00100000:1.0:1769074488.821627:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624662144 00000100:00100000:1.0:1769074488.821659:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624662272 00000100:00100000:1.0:1769074488.821662:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624662272 00000100:00100000:1.0:1769074488.821678:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 185 00000100:00100000:1.0:1769074488.821686:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33937:x1855007624661760:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.821700:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.821764:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074488.821770:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006136300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:33937:x1855007624661760:12345-192.168.204.102@tcp:104: Request processed in 85us (455us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.821777:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 185 00000100:00100000:1.0:1769074488.821794:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 186 00000100:00100000:1.0:1769074488.821800:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624661888:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.821807:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.821841:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074488.821846:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33938:x1855007624661888:12345-192.168.204.102@tcp:104: Request processed in 47us (476us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.821851:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 186 00000100:00100000:1.0:1769074488.821862:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 187 00000100:00100000:1.0:1769074488.821866:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:14023:x1855007624662144:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.821873:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.821899:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1769074488.821904:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:14023:x1855007624662144:12345-192.168.204.102@tcp:104: Request processed in 37us (509us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.821910:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 187 00000100:00100000:1.0:1769074488.821922:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 188 00000100:00100000:1.0:1769074488.821929:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b100602a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624662272:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074488.821935:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.821965:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1769074488.821971:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b100602a450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:33935:x1855007624662272:12345-192.168.204.102@tcp:104: Request processed in 43us (551us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074488.821977:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 188 00010000:00010000:1.0:1769074488.822009:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1769074488.822016:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822029:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074488.822034:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822075:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822083:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074488.822101:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822108:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d7ecec lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab32 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822129:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1769074488.822133:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822139:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699200 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074488.822142:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822162:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822169:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074488.822176:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822184:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d7ecd7 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aafa expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822196:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074488.822199:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822205:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b100164b800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074488.822208:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822224:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822230:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074488.822236:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822242:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164b800/0x68a7113811d7ed40 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac12 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822254:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074488.822257:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822262:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005deca00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:1.0:1769074488.822264:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822278:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822283:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074488.822287:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822293:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005deca00/0x68a7113811d7ed16 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305aba2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074488.822303:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074488.822304:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074488.822306:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074488.822313:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074488.822319:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074488.822340:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369408:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.822394:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369664:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.822441:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612367488:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.822462:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074488.822467:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369280:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.822493:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369536:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074488.822542:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074488.822867:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612366848:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.822896:0:13990:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1769074488.822907:0:13990:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624662400 00000100:00100000:2.0:1769074488.822911:0:13990:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624662400 00000100:00100000:2.0:1769074488.822951:0:13990:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 41 00000100:00100000:2.0:1769074488.822959:0:13990:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b10096a9050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33950:x1855007624662400:12345-192.168.204.102@tcp:104: 00010000:00010000:2.0:1769074488.822971:0:13990:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.823024:0:13990:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1769074488.823030:0:13990:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b10096a9050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:33950:x1855007624662400:12345-192.168.204.102@tcp:104: Request processed in 72us (1550us total) trans 0 rc 0/0 00000100:00100000:2.0:1769074488.823036:0:13990:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 41 00000100:00100000:2.0:1769074488.823097:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612367232:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.823124:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612367360:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074488.823173:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074488.823179:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.823187:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005cac000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.823192:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.823223:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.823230:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.823244:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.823252:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005cac000/0x68a7113811d7e647 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305997a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.823267:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.823269:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.823270:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074488.823318:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369792:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.838101:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074488.838199:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612368128:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.838331:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368512:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.838343:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368768:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.841866:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624662656 00000100:00100000:0.0:1769074488.841881:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624662656 00000100:00100000:0.0:1769074488.841946:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 189 00000100:00100000:0.0:1769074488.841955:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624662656:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.841973:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.842068:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074488.842075:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33942:x1855007624662656:12345-192.168.204.102@tcp:104: Request processed in 121us (3987us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.842082:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 189 00010000:00010000:3.0:1769074488.842357:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074488.842365:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.842373:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005657c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:3.0:1769074488.842380:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.842426:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.842434:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074488.845453:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.845474:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657c00/0x68a7113811d7ed6a lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac82 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074488.845500:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.845503:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074488.845505:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074488.845540:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612369920:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.848821:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.848849:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.848867:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.849128:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.849144:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.849163:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074488.849209:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369280:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.849223:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369536:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.849234:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369408:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.849243:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612369664:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.849289:0:13987:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074488.849306:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624663168 00000100:00100000:0.0:1769074488.849312:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624663168 00000100:00100000:0.0:1769074488.849348:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624663296 00000100:00100000:0.0:1769074488.849351:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624663296 00000100:00100000:0.0:1769074488.849361:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624663424 00000100:00100000:0.0:1769074488.849363:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624663424 00000100:00100000:0.0:1769074488.849371:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624663936 00000100:00100000:0.0:1769074488.849373:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624663936 00000100:00100000:0.0:1769074488.849380:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624664064 00000100:00100000:0.0:1769074488.849382:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624664064 00000100:00100000:0.0:1769074488.849390:0:13987:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624664192 00000100:00100000:0.0:1769074488.849392:0:13987:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624664192 00000100:00100000:0.0:1769074488.849405:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 190 00000100:00100000:0.0:1769074488.849412:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006024e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+8:33936:x1855007624663168:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.849426:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074488.849778:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612368640:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074488.850021:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074488.850029:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006024e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+8:33936:x1855007624663168:12345-192.168.204.102@tcp:104: Request processed in 617us (1220us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.850037:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 190 00000100:00100000:0.0:1769074488.850059:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 191 00000100:00100000:0.0:1769074488.850068:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33943:x1855007624663296:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.850078:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074488.850178:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369792:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.850190:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612368896:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.850201:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369152:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.850210:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612369024:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074488.850220:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612368384:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074488.858172:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074488.858190:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858203:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005656800 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.858210:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858301:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858310:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.858336:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858348:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656800/0x68a7113811d7ed2b lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305abda expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858366:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074488.858371:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858377:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005657600 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:2.0:1769074488.858381:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858402:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 2/0,0 mode: --/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858407:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074488.858413:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858419:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005657600/0x68a7113811d7ed94 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acf2 expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074488.858432:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.858434:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.858436:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074488.858460:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612370176:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.858580:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612369920:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.858601:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074488.858608:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612370048:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074488.859536:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1769074488.859552:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006026d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:33943:x1855007624663296:12345-192.168.204.102@tcp:104: Request processed in 9482us (10697us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.859564:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 191 00000100:00100000:0.0:1769074488.859608:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 192 00000100:00100000:0.0:1769074488.859624:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:33939:x1855007624663424:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.859658:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.859716:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074488.859722:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+7:33939:x1855007624663424:12345-192.168.204.102@tcp:104: Request processed in 105us (10853us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.859729:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 192 00000100:00100000:0.0:1769074488.859742:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 193 00000100:00100000:0.0:1769074488.859750:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006027480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33938:x1855007624663936:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.859758:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.859788:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074488.859793:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006027480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:33938:x1855007624663936:12345-192.168.204.102@tcp:104: Request processed in 44us (10666us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.859799:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 193 00000100:00100000:0.0:1769074488.859813:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 194 00000100:00100000:0.0:1769074488.859820:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6522:x1855007624664064:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.859828:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.859854:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074488.859860:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:6522:x1855007624664064:12345-192.168.204.102@tcp:104: Request processed in 41us (10715us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.859865:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 194 00000100:00100000:0.0:1769074488.859876:0:13987:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 195 00000100:00100000:0.0:1769074488.859883:0:13987:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1007c42c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6519:x1855007624664192:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074488.859889:0:13987:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.859923:0:13987:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:0.0:1769074488.859929:0:13987:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1007c42c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:6519:x1855007624664192:12345-192.168.204.102@tcp:104: Request processed in 47us (10766us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074488.859934:0:13987:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 195 00010000:00010000:0.0:1769074488.860119:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:0.0:1769074488.860125:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860133:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698c00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.860139:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860176:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860182:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.860208:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860214:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005698c00/0x68a7113811d7ed01 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ab6a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860244:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074488.860248:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860254:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005657400 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.860257:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860280:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860285:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.860292:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860298:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657400/0x68a7113811d7eda9 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ad2a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860311:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074488.860314:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860320:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005657000 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.860323:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860337:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860344:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.860347:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860353:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005657000/0x68a7113811d7ed7f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305acba expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860368:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074488.860371:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860376:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005656a00 already unused, calling callback (ffffffffc109c1b0) 00010000:00010000:0.0:1769074488.860379:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860391:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860397:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074488.860402:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860409:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005656a00/0x68a7113811d7ed55 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0x143201ebf305ac4a expref: -99 pid: 29778 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074488.860420:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.860421:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.860422:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074488.860436:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370304:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.860485:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370560:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.860510:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074488.860515:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370432:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.860555:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370688:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074488.860633:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.861147:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074488.861173:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074488.871231:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074488.872291:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370432:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.872306:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370688:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.872315:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370304:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074488.872322:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612370560:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.872410:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612370048:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074488.882609:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612370176:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074489.889034:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074490.912599:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074491.935842:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1769074492.536252:0:31516:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 6a: mkdir + contained create ======= 04:34:50 (1769074490) 00000080:00200000:3.0:1769074492.717090:0:31528:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074492.717118:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.717175:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.717186:0:31528:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.717195:0:31528:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31528 initiator: MDT0 00000100:00100000:3.0:1769074492.717222:0:31528:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612370816:192.168.204.102@tcp:101:lfs.0 00000100:00100000:3.0:1769074492.717306:0:31528:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074492.746679:0:31528:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612370816:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074492.746708:0:31528:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746722:0:31528:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746734:0:31528:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746746:0:31528:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746758:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746766:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746775:0:31528:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074492.746786:0:31528:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de680 x1855007612370816/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074508 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074492.746799:0:31528:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746813:0:31528:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074492.746820:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074492.746850:0:31528:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074492.746856:0:31528:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f9 00000080:00010000:3.0:1769074492.746861:0:31528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074492.746865:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3b58 00010000:00010000:3.0:1769074492.746868:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.746875:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.746900:0:31528:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074492.746928:0:31528:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=13 00000080:00200000:3.0:1769074492.747313:0:31528:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074492.747337:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.747382:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.747390:0:31528:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.747399:0:31528:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31528 initiator: MDT0 00000100:00100000:3.0:1769074492.747419:0:31528:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612370944:192.168.204.102@tcp:101:lfs.0 00000100:00100000:3.0:1769074492.747550:0:31528:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074492.754835:0:31528:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612370944:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074492.754858:0:31528:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754871:0:31528:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754878:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754887:0:31528:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754895:0:31528:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754901:0:31528:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169a00 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:3.0:1769074492.754906:0:31528:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754911:0:31528:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754917:0:31528:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754923:0:31528:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074492.754928:0:31528:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 31528 initiator: MDT0 00010000:00010000:3.0:1769074492.754945:0:31528:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80400 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 31528 initiator: MDT0 00000002:00100000:3.0:1769074492.754962:0:31528:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b10037de680 x1855007612370944/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074508 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1769074492.754975:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1769074492.755000:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755016:0:31528:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00010000:3.0:1769074492.755024:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3940 00010000:00010000:3.0:1769074492.755028:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755037:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.755121:0:31528:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074492.755134:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.755141:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755148:0:31528:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.755154:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074492.755158:0:31528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074492.755162:0:31528:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f9 00000080:00010000:3.0:1769074492.755167:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3be8 00010000:00010000:3.0:1769074492.755170:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755412:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.755426:0:31528:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074492.755436:0:31528:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6a.replay-single, flags=16392 00000080:00200008:3.0:1769074492.755476:0:31528:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074492.755481:0:31528:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074492.755489:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.755496:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755504:0:31528:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.755509:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074492.755512:0:31528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074492.755515:0:31528:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f9 00000080:00010000:3.0:1769074492.755518:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3cb8 00010000:00010000:3.0:1769074492.755520:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.755526:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074492.756337:0:31528:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074492.756343:0:31528:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074492.756353:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.756362:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756371:0:31528:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.756377:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074492.756381:0:31528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074492.756383:0:31528:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f9 00000080:00010000:3.0:1769074492.756387:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3cb8 00010000:00010000:3.0:1769074492.756390:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756396:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.756533:0:31528:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074492.756541:0:31528:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074492.756548:0:31528:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756556:0:31528:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.756562:0:31528:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074492.756565:0:31528:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074492.756568:0:31528:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d803f9 00000080:00010000:3.0:1769074492.756571:0:31528:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d803f9 from it ffffaa9141cf3bb0 00010000:00010000:3.0:1769074492.756574:0:31528:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756581:0:31528:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074492.756594:0:31528:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074492.756601:0:31528:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00000080:00200080:3.0:1769074492.756617:0:31528:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c03066f0 arg=7ffed45486c0 00000080:00200000:3.0:1769074492.756630:0:31528:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) name=d6a.replay-single stripe_offset=0 stripe_count=1, hash_type=0 00010000:00010000:3.0:1769074492.756671:0:31528:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756724:0:31528:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074492.756731:0:31528:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074492.756734:0:31528:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d803f9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062b8c expref: -99 pid: 31528 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074492.756755:0:31528:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074492.756766:0:31528:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de680 x1855007612371200/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074492.756788:0:31528:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612371200:192.168.204.102@tcp:36:lfs.0 00000100:00100000:3.0:1769074492.756865:0:31528:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074492.767815:0:31528:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31528:1855007612371200:192.168.204.102@tcp:36:lfs.0 00000080:00200000:3.0:1769074492.767861:0:31528:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08) 00000080:00200000:3.0:1769074492.767869:0:31528:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 512 00000080:00200008:3.0:1769074492.767909:0:31528:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00010000:00010000:0.0:1769074492.959886:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074493.088108:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074493.088197:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074493.088230:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074493.088236:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612371328:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074493.088264:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371584:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074493.095397:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074493.103074:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612371328:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074493.103107:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371584:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074493.103121:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612371712:192.168.204.102@tcp:400:kworker.0 00000080:00200008:3.0:1769074493.868740:0:31538:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074493.868754:0:31538:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074493.868775:0:31538:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074493.868837:0:31538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 31538 initiator: MDT0 00010000:00010000:3.0:1769074493.868846:0:31538:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31538 initiator: MDT0 00010000:00010000:3.0:1769074493.868857:0:31538:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31538 initiator: MDT0 00000100:00100000:3.0:1769074493.868882:0:31538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612371840:192.168.204.102@tcp:101:lfs.0 00000100:00100000:3.0:1769074493.868965:0:31538:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074493.877722:0:31538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612371840:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074493.877748:0:31538:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877761:0:31538:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877772:0:31538:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877782:0:31538:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877794:0:31538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877801:0:31538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877808:0:31538:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074493.877819:0:31538:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc000 x1855007612371840/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074509 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074493.877833:0:31538:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877845:0:31538:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074493.877851:0:31538:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074493.877878:0:31538:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074493.877885:0:31538:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074493.877890:0:31538:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074493.877894:0:31538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141cf3cb8 00010000:00010000:3.0:1769074493.877897:0:31538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.877905:0:31538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074493.878909:0:31538:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074493.879461:0:31538:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074493.879481:0:31538:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074493.879496:0:31538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.879511:0:31538:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074493.879518:0:31538:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074493.879525:0:31538:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074493.879528:0:31538:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074493.879534:0:31538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141cf3cb8 00010000:00010000:3.0:1769074493.879537:0:31538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.879545:0:31538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074493.880192:0:31538:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074493.880204:0:31538:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074493.880213:0:31538:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.880223:0:31538:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074493.880230:0:31538:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074493.880234:0:31538:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074493.880237:0:31538:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074493.880240:0:31538:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141cf3bb0 00010000:00010000:3.0:1769074493.880244:0:31538:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074493.880250:0:31538:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074493.880452:0:31538:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074493.880463:0:31538:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074493.880633:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.880669:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00800000:00000004:3.0:1769074493.880677:0:31538:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1458, max_age 1486 00000100:00100000:3.0:1769074493.880720:0:31538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612371968:192.168.204.102@tcp:41:lfs.0 00000100:00100000:3.0:1769074493.880808:0:31538:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074493.885952:0:31538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612371968:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074493.885988:0:31538:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500643/500989 00000080:00200080:3.0:1769074493.886060:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886077:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886084:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886089:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886093:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886097:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886101:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886104:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886108:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886111:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886115:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886118:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886122:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886125:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886129:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886132:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886136:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886139:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886144:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886148:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886151:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886154:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886157:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886160:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886163:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886166:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886169:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886172:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886176:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886179:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886182:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886185:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886188:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886192:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886195:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886199:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886202:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886206:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886209:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886212:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886215:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886218:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886221:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886224:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886228:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886231:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886235:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886238:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886241:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886244:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886247:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886250:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886253:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886257:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886261:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886265:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886268:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886271:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886274:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886277:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886280:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886283:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886287:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886291:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886295:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886298:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886302:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886306:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886310:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886313:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886317:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886319:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886324:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886327:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886330:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886334:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886338:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886341:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886344:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886347:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886351:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886354:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886357:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886361:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886364:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886367:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886370:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886373:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886376:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886379:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886383:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886385:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886389:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886392:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886396:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886399:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886402:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886406:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886410:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886414:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886418:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886421:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886425:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886428:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886432:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886434:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886451:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886455:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886459:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886462:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886466:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886469:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886473:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886476:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886480:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886483:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886487:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886490:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886494:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886498:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886501:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886505:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886509:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886512:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886515:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886518:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886522:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00800000:00000080:3.0:1769074493.886525:0:31538:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.886530:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.886534:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00020000:00000004:3.0:1769074493.886542:0:31538:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1458, max_age 1486 00000100:00100000:3.0:1769074493.886847:0:31538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612372096:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074493.886923:0:31538:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074493.895427:0:31538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612372096:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074493.895469:0:31538:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074493.895545:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.895562:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00020000:00000004:3.0:1769074493.895569:0:31538:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1458, max_age 1486 00000100:00100000:3.0:1769074493.895614:0:31538:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612372224:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074493.895716:0:31538:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074493.900682:0:31538:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31538:1855007612372224:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074493.900714:0:31538:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074493.900782:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900799:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900806:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900810:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900813:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900816:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900820:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900823:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900826:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900830:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900833:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900836:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900840:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900843:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900847:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900851:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900855:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900858:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900862:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900866:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900869:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900873:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900876:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900879:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900882:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900899:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900904:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900908:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900911:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900915:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900918:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900922:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900925:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900929:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900933:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900936:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900940:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900943:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900946:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900950:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900953:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900956:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900960:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900963:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900967:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900970:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900973:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900977:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900981:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900984:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900988:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.900992:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.900996:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901000:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901003:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901006:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901010:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901013:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901017:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901020:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901024:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901027:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901031:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901034:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901037:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901041:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901045:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901048:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901051:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901054:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901059:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901074:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901080:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901083:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901087:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901090:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901093:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901097:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901100:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901103:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901106:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901109:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901113:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901116:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901119:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901122:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901126:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901129:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901132:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901135:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901138:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901142:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901145:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901148:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901152:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901155:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901158:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901162:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901165:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901168:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901172:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901174:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901178:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901180:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901184:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901187:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901190:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901193:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901197:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901200:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901203:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901207:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901210:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901214:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901217:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901220:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901224:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901227:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901230:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901233:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901236:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901240:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901243:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901246:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200080:3.0:1769074493.901250:0:31538:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc484bdde0 00020000:00000080:3.0:1769074493.901254:0:31538:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35c00 uarg=0000000000000000 00000080:00200008:3.0:1769074493.901267:0:31538:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:3.0:1769074493.983873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1769074494.018219:0:31545:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074494.018247:0:31545:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074494.018268:0:31545:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018286:0:31545:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018294:0:31545:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074494.018302:0:31545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074494.018307:0:31545:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074494.018314:0:31545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141837bd0 00010000:00010000:3.0:1769074494.018318:0:31545:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018327:0:31545:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018344:0:31545:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074494.018359:0:31545:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:3.0:1769074494.018362:0:31545:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074494.018368:0:31545:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074494.018375:0:31545:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018382:0:31545:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018388:0:31545:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074494.018392:0:31545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074494.018394:0:31545:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074494.018397:0:31545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141837cd0 00010000:00010000:3.0:1769074494.018400:0:31545:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018406:0:31545:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018413:0:31545:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074494.018418:0:31545:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074494.018423:0:31545:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074494.018427:0:31545:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018448:0:31545:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018455:0:31545:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074494.018460:0:31545:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074494.018462:0:31545:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80407 00000080:00010000:3.0:1769074494.018466:0:31545:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80407 from it ffffaa9141837db0 00010000:00010000:3.0:1769074494.018468:0:31545:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018475:0:31545:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.018483:0:31545:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074494.018488:0:31545:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:3.0:1769074494.018492:0:31545:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:3.0:1769074494.018520:0:31545:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018577:0:31545:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.018584:0:31545:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074494.018587:0:31545:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80407 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062bbd expref: -99 pid: 31538 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074494.018613:0:31545:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074494.018627:0:31545:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc000 x1855007612372480/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074494.018677:0:31545:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31545:1855007612372480:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:3.0:1769074494.018775:0:31545:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074494.026987:0:31545:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477398 gen 1 00000100:00100000:3.0:1769074494.027028:0:31545:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037dc000 x1855007612372480/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/448 e 0 to 0 dl 1769074510 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074494.027043:0:31545:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31545:1855007612372480:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:3.0:1769074494.027100:0:31545:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x108:0x0](ffff8b100485db88) 00000080:00200000:3.0:1769074494.027142:0:31545:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:1.0:1769074494.056314:0:31548:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074494.056337:0:31548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074494.056391:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 31548 initiator: MDT0 00010000:00010000:1.0:1769074494.056401:0:31548:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31548 initiator: MDT0 00010000:00010000:1.0:1769074494.056414:0:31548:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31548 initiator: MDT0 00000100:00100000:1.0:1769074494.056448:0:31548:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372608:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074494.056527:0:31548:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10090219c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074494.087761:0:31548:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372608:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074494.087789:0:31548:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087801:0:31548:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087813:0:31548:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087824:0:31548:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087835:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087842:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087849:0:31548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074494.087859:0:31548:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006137480 x1855007612372608/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074510 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074494.087872:0:31548:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087885:0:31548:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074494.087892:0:31548:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074494.087933:0:31548:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074494.087940:0:31548:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8040e 00000080:00010000:3.0:1769074494.087946:0:31548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074494.087949:0:31548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8040e from it ffffaa9141c43b58 00010000:00010000:3.0:1769074494.087952:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.087960:0:31548:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074494.087984:0:31548:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074494.088006:0:31548:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:3.0:1769074494.088012:0:31548:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074494.088034:0:31548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074494.088095:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 31548 initiator: MDT0 00010000:00010000:3.0:1769074494.088103:0:31548:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31548 initiator: MDT0 00010000:00010000:3.0:1769074494.088113:0:31548:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31548 initiator: MDT0 00000100:00100000:3.0:1769074494.088140:0:31548:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372736:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074494.088222:0:31548:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074494.092620:0:31548:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372736:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074494.092683:0:31548:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092706:0:31548:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092715:0:31548:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092727:0:31548:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092737:0:31548:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092915:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092924:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092931:0:31548:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074494.092942:0:31548:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de680 x1855007612372736/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074510 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074494.092955:0:31548:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074494.092966:0:31548:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/264 (0 0) 00000080:00200000:3.0:1769074494.092974:0:31548:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074494.094026:0:31548:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x108:0x0], updating i_size 0 00000080:00200000:2.0:1769074494.094034:0:31548:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x108:0x0](ffff8b100485db88): rc = 0 00000080:00200000:2.0:1769074494.094051:0:31548:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x108:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1769074494.094423:0:31548:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x108:0x0](ffff8b100485db88) for lock 0x68a7113811d80415 00000080:00010000:2.0:1769074494.094448:0:31548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x108:0x0](ffff8b100485db88) 00000080:00010000:2.0:1769074494.094453:0:31548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80415 from it ffffaa9141c43b20 00010000:00010000:2.0:1769074494.094458:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.094469:0:31548:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074494.094510:0:31548:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x108:0x0](ffff8b100485db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074494.094517:0:31548:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x108:0x0](ffff8b100485db88),name=fsa-oleg402-client.virtnet 00000002:00010000:2.0:1769074494.094539:0:31548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x108:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074494.094550:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.094560:0:31548:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.094567:0:31548:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074494.094573:0:31548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x108:0x0](ffff8b100485db88) 00000080:00010000:2.0:1769074494.094576:0:31548:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x108:0x0](ffff8b100485db88) for lock 0x68a7113811d80415 00000080:00010000:2.0:1769074494.094579:0:31548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80415 from it ffffaa9141c43cb8 00010000:00010000:2.0:1769074494.094582:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.094588:0:31548:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095261:0:31548:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074494.095275:0:31548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074494.095334:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095345:0:31548:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095352:0:31548:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074494.095358:0:31548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074494.095485:0:31548:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8040e 00000080:00010000:2.0:1769074494.095491:0:31548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8040e from it ffffaa9141c43c10 00010000:00010000:2.0:1769074494.095494:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095502:0:31548:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095517:0:31548:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074494.095528:0:31548:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:2.0:1769074494.095533:0:31548:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074494.095539:0:31548:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074494.095545:0:31548:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095553:0:31548:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095559:0:31548:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074494.095562:0:31548:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074494.095565:0:31548:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8040e 00000080:00010000:2.0:1769074494.095568:0:31548:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8040e from it ffffaa9141c43d68 00010000:00010000:2.0:1769074494.095571:0:31548:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095578:0:31548:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095584:0:31548:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074494.095588:0:31548:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074494.095610:0:31548:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095632:0:31548:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074494.095661:0:31548:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x108:0x0]: losing layout lock 00010000:00010000:2.0:1769074494.095691:0:31548:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095698:0:31548:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095704:0:31548:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074494.095707:0:31548:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168000/0x68a7113811d80415 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3062bf5 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074494.095723:0:31548:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9400/0x68a7113811d8040e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062bd9 expref: -99 pid: 31548 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074494.095740:0:31548:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074494.096483:0:31548:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5b480 x1855007612372992/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074494.096515:0:31548:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372992:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074494.096754:0:31548:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074494.131243:0:31548:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:31548:1855007612372992:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074494.131458:0:31548:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x108:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074494.131470:0:31548:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x108:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074494.131481:0:31548:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x108:0x0](ffff8b100485db88) 00010000:00010000:2.0:1769074495.008146:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074496.031936:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074497.062019:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1769074497.676447:0:31579:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:1.0:1769074497.697556:0:31580:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074497.697597:0:31580:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074497.699469:0:31580:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 31580 initiator: MDT0 00010000:00010000:1.0:1769074497.699492:0:31580:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31580 initiator: MDT0 00010000:00010000:1.0:1769074497.699505:0:31580:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31580 initiator: MDT0 00000100:00100000:1.0:1769074497.699544:0:31580:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373120:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:1.0:1769074497.699694:0:31580:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009021f00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074497.709308:0:31580:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373120:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:1.0:1769074497.709346:0:31580:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709362:0:31580:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709386:0:31580:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709400:0:31580:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709414:0:31580:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709424:0:31580:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709432:0:31580:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074497.709445:0:31580:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006136d80 x1855007612373120/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074513 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074497.709461:0:31580:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709476:0:31580:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074497.709485:0:31580:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074497.709525:0:31580:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074497.709532:0:31580:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8041c 00000080:00010000:1.0:1769074497.709539:0:31580:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074497.709542:0:31580:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8041c from it ffffaa9141ca3bd0 00010000:00010000:1.0:1769074497.709545:0:31580:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.709554:0:31580:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074497.709584:0:31580:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074497.709609:0:31580:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=24 00000080:00200000:1.0:1769074497.709615:0:31580:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=0 00000002:00010000:1.0:1769074497.709680:0:31580:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:1.0:1769074497.709736:0:31580:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 31580 initiator: MDT0 00010000:00010000:1.0:1769074497.709744:0:31580:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31580 initiator: MDT0 00010000:00010000:1.0:1769074497.709754:0:31580:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 31580 initiator: MDT0 00000100:00100000:1.0:1769074497.709780:0:31580:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373248:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:1.0:1769074497.709879:0:31580:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009021f00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074497.716988:0:31580:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373248:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:1.0:1769074497.717024:0:31580:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717052:0:31580:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717061:0:31580:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717076:0:31580:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717091:0:31580:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717111:0:31580:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717121:0:31580:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717129:0:31580:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074497.717144:0:31580:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8b1006136d80 x1855007612373248/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074513 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074497.717160:0:31580:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717173:0:31580:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/263 (0 0) 00000080:00200000:1.0:1769074497.717187:0:31580:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074497.717218:0:31580:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 11776 00000080:00200000:1.0:1769074497.717222:0:31580:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x107:0x0](ffff8b100485cb08): rc = 0 00000080:00010000:1.0:1769074497.717229:0:31580:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) for lock 0x68a7113811d80423 00000080:00010000:1.0:1769074497.717244:0:31580:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) 00000080:00010000:1.0:1769074497.717248:0:31580:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80423 from it ffffaa9141ca3aa0 00010000:00010000:1.0:1769074497.717251:0:31580:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074497.717261:0:31580:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074497.717308:0:31580:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074497.717335:0:31580:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffff8b100485cb08), flags=1536 00000080:00200000:1.0:1769074497.717339:0:31580:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074497.717346:0:31580:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074497.717351:0:31580:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffff8b100485cb08), flags=33188, excl=1 00000080:00200000:1.0:1769074497.717356:0:31580:0:(namei.c:2020:ll_mknod()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffff8b100485cb08) mode 100644 dev 0 00000020:00100000:1.0:1769074497.717448:0:31580:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074497.717457:0:31580:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006136d80 x1855007612373504/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074497.717485:0:31580:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373504:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:1.0:1769074497.717587:0:31580:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009021f00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074497.733308:0:31580:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:31580:1855007612373504:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074497.733388:0:31580:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x109:0x0](ffff8b100486ec08) 00000080:00200000:2.0:1769074497.733444:0:31580:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f6a.replay-single, unhashed 1 00010000:00010000:3.0:1769074498.080717:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074498.207887:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612373760:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.208035:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612374016:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.208074:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074498.208081:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612373632:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.208116:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612373888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.215404:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612373888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.215433:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612373760:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.215447:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612374016:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074498.242677:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612373632:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074499.104128:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074500.127973:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074501.153472:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074502.180722:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074503.200478:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074503.328194:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374272:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074503.328307:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374528:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074503.328347:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074503.328354:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612374144:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074503.328389:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374400:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074503.338277:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374400:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074503.338309:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374528:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074504.223883:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074505.248563:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074506.271958:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074507.296692:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074508.321505:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074508.447906:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374784:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074508.448005:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375040:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074508.448051:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074508.448064:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612374656:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074508.448116:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374912:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074508.451506:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375040:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074508.464397:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374912:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074509.343890:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074510.368183:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074511.392896:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074512.415952:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074513.440608:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074513.571424:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375296:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074513.571562:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375552:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074513.571603:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074513.571610:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612375168:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074513.571665:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375424:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074513.577520:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375424:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074513.587157:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375552:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074514.463968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074515.488322:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074516.511929:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074517.542525:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074518.560071:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074518.688949:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612375680:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074518.689118:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612375936:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074518.689176:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1769074518.689185:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612375808:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074518.689217:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612376064:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074518.696254:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612376064:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074518.696279:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612375936:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074519.584881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074519.712190:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074503/real 1769074503] req@ffff8b1005d58e00 x1855007612374144/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074519 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074519.712236:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58e00 x1855007612374144/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074519 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074519.712247:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1769074519.712252:0:2366:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1769074519.712256:0:2366:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1769074519.712259:0:2366:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1769074519.712263:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d58e00 x1855007612374144/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074519 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074519.712273:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5ad80 x1855007612374656/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074524 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074519.712281:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d59f80 x1855007612375168/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074529 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074519.712289:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5a680 x1855007612375680/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074534 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1769074519.712317:0:2366:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1769074519.712321:0:2366:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1769074519.712322:0:2366:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1769074519.712324:0:2366:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:2.0:1769074519.712353:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074503/real 1769074503] req@ffff8b1005d5bb80 x1855007612374272/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074519 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074519.712377:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d5bb80 x1855007612374272/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074519 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074519.712392:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1769074519.712396:0:2366:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1769074519.714238:0:16087:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074519.714250:0:16087:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074519.714256:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074519.714267:0:16087:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1440 00000100:00080000:0.0:1769074519.714287:0:16087:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769074519.714350:0:16087:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025500 x1855007612376192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074519.714866:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612376192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769074519.716296:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612375680:192.168.204.102@tcp:400:kworker.0 00000100:00080000:3.0:1769074519.730408:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:3.0:1769074519.730434:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1769074519.730446:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf305c8b3 to 0x143201ebf3062c6c 00000100:00080000:3.0:1769074519.730452:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1769074519.730459:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:1.0:1769074519.734864:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612376192:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074519.735694:0:31767:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:3.0:1769074519.735703:0:31767:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 13s for inflight to error out 00000100:00080000:2.0:1769074519.751655:0:2366:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1769074519.754127:0:16087:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074519.754140:0:16087:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074519.754146:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074519.754158:0:16087:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1440 00000100:00080000:0.0:1769074519.754180:0:16087:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074519.754198:0:16087:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074519.754246:0:16087:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612376320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074519.754389:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612376448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074519.754506:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074519.754514:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612376576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074519.759222:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612376576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074519.759247:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612376448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074519.759681:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612376320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769074519.762731:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612375168:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074519.762765:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612374656:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074519.762772:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612374144:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074519.762780:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374272:192.168.204.102@tcp:400:kworker.0 10000000:01000000:3.0:1769074519.763269:0:31767:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1769074519.763291:0:31767:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006171400/0x68a7113811d7f7a4 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf305d11f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074519.763311:0:31767:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006171400/0x68a7113811d7f7a4 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf305d11f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074519.763318:0:31767:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074519.763337:0:31767:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074519.763350:0:31767:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006171400/0x68a7113811d7f7a4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf305d11f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074519.763369:0:31767:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b100602c200/0x68a7113811d7f7b2 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf305d181 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074519.763376:0:31767:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b100602c200/0x68a7113811d7f7b2 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf305d181 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074519.763568:0:31767:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074519.763573:0:31767:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074519.763577:0:31767:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b100602c200/0x68a7113811d7f7b2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf305d181 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074519.763598:0:31767:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b100601ce00/0x68a7113811d7f79d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf305d0fc expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074519.763610:0:31767:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b100601ce00/0x68a7113811d7f79d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf305d0fc expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074519.763615:0:31767:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074519.763619:0:31767:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074519.763622:0:31767:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b100601ce00/0x68a7113811d7f79d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf305d0fc expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074519.763890:0:31767:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006170000/0x68a7113811d7f7ab lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf305d142 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074519.763896:0:31767:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006170000/0x68a7113811d7f7ab lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf305d142 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074519.763901:0:31767:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1769074519.763912:0:31767:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074519.763917:0:31767:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006170000/0x68a7113811d7f7ab lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf305d142 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:3.0:1769074519.763935:0:31767:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1769074519.763949:0:31767:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1769074519.763952:0:31767:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1769074519.763956:0:31767:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1769074519.763962:0:31767:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:3.0:1769074520.607943:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1769074521.119854:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074521.119864:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 22 10000000:01000000:2.0:1769074521.119872:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074521.119946:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.119960:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.119974:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074521.120020:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376704:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.120198:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.131696:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376704:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074521.131739:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.131754:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074521.131860:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376832:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.131972:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.182860:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376832:192.168.204.102@tcp:256:ll_cfg_requeue.0 10000000:01000000:2.0:1769074521.182939:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1769074521.182953:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.182987:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074521.183001:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1769074521.183004:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:2.0:1769074521.183008:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1769074521.183054:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.183064:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.183075:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074521.183101:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376960:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.183209:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.187252:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612376960:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074521.187300:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.187317:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074521.187428:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377088:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.187533:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.192211:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377088:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.192324:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377216:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.192455:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.205144:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377216:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1769074521.206430:0:31768:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:3.0:1769074521.209229:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:3.0:1769074521.209371:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1769074521.209390:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.209407:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074521.209419:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1769074521.209422:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:3.0:1769074521.209426:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1769074521.209484:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.209494:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.209508:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074521.209769:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377344:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074521.210033:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ece40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.216245:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377344:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074521.216296:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074521.216312:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074521.216423:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377472:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.216535:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.221398:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377472:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.221493:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377600:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074521.221676:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074521.225683:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377600:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1769074521.227802:0:31769:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1769074521.228882:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1769074521.229108:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1769074521.229123:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074521.229140:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074521.229154:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1769074521.229159:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:0.0:1769074521.229162:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1769074521.229215:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074521.229224:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074521.229236:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074521.229272:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377728:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074521.229546:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c000 going to sleep for 11 seconds 00000100:00100000:3.0:1769074521.239462:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377728:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074521.239503:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.239519:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074521.239729:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377856:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074521.239899:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074521.253579:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612377856:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1769074521.253710:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1769074521.253723:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.253737:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074521.632003:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074522.655941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074523.684936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074524.709206:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074524.833190:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074524.833201:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769074524.833258:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612378112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074524.833376:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074524.833384:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612377984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074524.833423:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612378240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074524.853267:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612377984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074524.861801:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612378112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074524.874750:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612378240:192.168.204.102@tcp:400:kworker.0 00000100:00000400:2.0:1769074524.895832:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074508/real 1769074508] req@ffff8b1005d58a80 x1855007612374784/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074524 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074524.895881:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58a80 x1855007612374784/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074524 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074524.895891:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 48, was 47): CONNECTING 00000100:00100000:2.0:1769074524.895915:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612374784:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074525.729332:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074526.753286:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074527.776550:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074528.266053:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074528.266814:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624673408 00000100:00100000:0.0:1769074528.266830:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624673408 00000100:00100000:0.0:1769074528.266927:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 196 00000100:00100000:0.0:1769074528.266938:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:34840:x1855007624673408:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074528.266959:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074528.267063:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00010000:00010000:3.0:1769074528.267259:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074528.267267:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074528.267275:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699200 already unused, calling callback (ffffffffc121fc20) 10000000:00010000:3.0:1769074528.267280:0:13998:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074528.267289:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074528.267297:0:13998:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074528.267303:0:13998:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074528.267369:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074528.267376:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074528.267403:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074528.267410:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005699200/0x68a7113811d8042a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3062c73 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074528.267432:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074528.267434:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074528.267436:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074528.267459:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612378368:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074528.269405:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:34840:x1855007624673408:12345-192.168.204.102@tcp:104: Request processed in 135us (1035us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074528.269426:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 196 00000100:00100000:3.0:1769074528.300908:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612378368:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074528.800920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1769074529.631879:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074529.631895:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 22 10000000:01000000:2.0:1769074529.631903:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074529.631993:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074529.632007:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074529.632021:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074529.632061:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612378496:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074529.632196:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074529.644055:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612378496:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074529.644098:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074529.644113:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074529.644218:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612378624:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074529.644387:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074529.696035:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612378624:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:2.0:1769074529.696118:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:2.0:1769074529.696288:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1769074529.696301:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074529.696315:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:1.0:1769074529.784146:0:31782:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 48, was 0): CONNECTING 00010000:00080000:1.0:1769074529.784190:0:31782:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8b1008679800@lustre-MDT0000-mdc-ffff8b101127b800: found existing conn 192.168.204.102@tcp, moved to head 00010000:00010000:3.0:1769074529.824052:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074529.954978:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074519/real 1769074519] req@ffff8b1006024e00 x1855007612376320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074529 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074529.955019:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612376320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074529 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074529.955044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769074529.955055:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769074529.955060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612376320:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074529.955261:0:16087:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074529.955266:0:16087:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074529.955269:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074529.955275:0:16087:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 0 00000100:00080000:3.0:1769074529.955293:0:16087:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074529.955307:0:16087:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074529.955325:0:16087:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612378880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074529.955397:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612378752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074529.955526:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612379136:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074529.955565:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074529.955571:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612379008:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074529.955978:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612378880:192.168.204.102@tcp:38:kworker.0 00010000:00010000:3.0:1769074529.962283:0:2366:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1769074529.962310:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612379008:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074529.962328:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612378752:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074529.962404:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074529.962424:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074529.962426:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074529.962427:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074529.963245:0:2366:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:3.0:1769074529.963262:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612379136:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074529.963322:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074529.963336:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074529.963338:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074529.963339:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:1.0:1769074529.989980:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b10037df480 x1855007612378880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074539 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074529.990004:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477400 gen 1 00000100:00100000:1.0:1769074529.990032:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136d80 x1855007612373504/t68719477401(68719477401) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074513 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074529.990051:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 17 00000100:00080000:1.0:1769074529.990075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1769074529.990082:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074529.990084:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477400 gen 1 00000100:00100000:1.0:1769074529.990224:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136d80 x1855007612373504/t68719477401(68719477401) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074513 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074529.990233:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 68719477400 last 0 00000100:00080000:1.0:1769074529.990241:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1006136d80 x1855007612373504/t68719477401(68719477401) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074513 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074529.990257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612378880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074529.990289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612373504:192.168.204.102@tcp:36:mcreate.0 00000100:00080000:3.0:1769074530.058233:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1006136d80 x1855007612373504/t68719477401(68719477401) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074545 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074530.058254:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074530.058258:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477400 gen 1 00000100:00100000:3.0:1769074530.058263:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136d80 x1855007612373504/t68719477401(68719477401) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074545 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074530.058271:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 68719477400 last 68719477401 00000100:00080000:3.0:1769074530.058276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:0.0:1769074530.066039:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612373504:192.168.204.102@tcp:36:mcreate.0 00010000:00080000:2.0:1769074530.066289:0:31796:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:2.0:1769074530.066297:0:31796:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (2) 00010000:00010000:2.0:1769074530.066371:0:31796:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d80423 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3062c3b expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074530.066413:0:31796:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9600/0x68a7113811d8041c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3062c1f expref: -99 pid: 31580 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074530.066425:0:31796:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:2.0:1769074530.067193:0:31796:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:0.0:1769074530.067399:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612379264:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00080000:0.0:1769074530.202903:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:0.0:1769074530.202932:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1005d58a80 x1855007612379264/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074546 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074530.202945:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1005d58a80 x1855007612379264/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074546 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074530.202954:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:0.0:1769074530.203139:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00100000:0.0:1769074530.203147:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612379264:192.168.204.102@tcp:400:ldlm_lock_repla.0 00010000:00010000:0.0:1769074530.848189:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074530.981242:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074513/real 1769074513] req@ffff8b1005d58700 x1855007612375296/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074529 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074530.981303:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58700 x1855007612375296/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074529 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074530.981330:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612375296:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074531.872479:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074532.898069:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074533.925167:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074534.431787:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074518/real 1769074518] req@ffff8b1005d59180 x1855007612375808/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074534 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074534.431838:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d59180 x1855007612375808/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074534 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074534.431867:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612375808:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074534.949193:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074535.074821:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612379392:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.075200:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379648:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.075248:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074535.075257:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379520:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.075285:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379776:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.083124:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612379392:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.094206:0:2367:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 68719477401 gen 1 00000100:00100000:2.0:1769074535.094246:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379520:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.097745:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379776:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074535.100965:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612379648:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074535.970911:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074536.995419:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074538.017282:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074539.053179:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074539.092433:0:31872:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074539.092451:0:31872:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074539.092478:0:31872:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074539.092548:0:31872:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 31872 initiator: MDT0 00010000:00010000:0.0:1769074539.092559:0:31872:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31872 initiator: MDT0 00010000:00010000:0.0:1769074539.092568:0:31872:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 31872 initiator: MDT0 00000100:00100000:0.0:1769074539.092597:0:31872:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612379904:192.168.204.102@tcp:101:lfs.0 00000100:00100000:2.0:1769074539.096724:0:31872:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1003324840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074539.105916:0:31872:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612379904:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074539.105947:0:31872:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.105961:0:31872:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.105974:0:31872:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.105986:0:31872:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.105998:0:31872:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106008:0:31872:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106015:0:31872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074539.106026:0:31872:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006026d80 x1855007612379904/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074555 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074539.106041:0:31872:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106053:0:31872:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074539.106059:0:31872:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074539.106084:0:31872:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074539.106089:0:31872:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074539.106094:0:31872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074539.106097:0:31872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa914202fcb8 00010000:00010000:2.0:1769074539.106101:0:31872:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106109:0:31872:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074539.106917:0:31872:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074539.106925:0:31872:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074539.106940:0:31872:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074539.106950:0:31872:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106963:0:31872:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074539.106970:0:31872:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074539.106975:0:31872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074539.106979:0:31872:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074539.106983:0:31872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa914202fcb8 00010000:00010000:2.0:1769074539.106987:0:31872:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.106994:0:31872:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074539.107089:0:31872:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074539.107096:0:31872:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074539.107102:0:31872:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.107109:0:31872:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074539.107114:0:31872:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074539.107118:0:31872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074539.107120:0:31872:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074539.107124:0:31872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa914202fbb0 00010000:00010000:2.0:1769074539.107127:0:31872:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074539.107139:0:31872:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074539.107147:0:31872:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074539.107153:0:31872:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074539.107389:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.107403:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00800000:00000004:2.0:1769074539.107410:0:31872:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1487, max_age 1531 00000100:00100000:2.0:1769074539.108770:0:31872:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380032:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074539.108872:0:31872:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074539.122033:0:31872:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380032:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074539.122066:0:31872:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:2.0:1769074539.122151:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122170:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122176:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122180:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122184:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122188:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122193:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122197:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122202:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122205:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122209:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122212:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122218:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122222:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122226:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122230:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122235:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122238:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122242:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122246:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122250:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122253:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122257:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122260:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122264:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122267:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122270:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122273:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122276:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122279:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122282:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122286:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122290:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122293:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122296:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122299:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122302:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122305:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122309:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122312:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122315:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122319:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122322:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122325:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122329:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122332:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122335:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122338:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122342:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122345:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122349:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122353:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122356:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122360:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122363:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122365:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122369:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122372:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122376:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122381:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122386:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122390:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122396:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122404:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122409:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122414:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122418:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122422:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122427:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122430:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122434:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122438:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122456:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122461:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122465:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122472:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122476:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122481:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122486:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122489:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122494:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122497:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122501:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122504:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122510:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122514:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122520:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122524:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122529:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122532:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122536:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122540:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122546:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122551:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122555:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122558:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122566:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122570:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122574:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122578:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122582:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122585:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122589:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122592:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122597:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122601:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122605:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122608:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122612:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122615:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122620:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122623:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122628:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122630:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122634:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122677:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122693:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122698:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122701:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122704:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122707:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122710:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122714:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122717:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122720:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122723:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122726:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00800000:00000080:2.0:1769074539.122729:0:31872:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.122733:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.122738:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00020000:00000004:2.0:1769074539.122746:0:31872:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1487, max_age 1531 00000100:00100000:2.0:1769074539.122786:0:31872:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380160:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074539.122867:0:31872:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074539.127821:0:31872:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380160:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074539.127851:0:31872:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074539.127925:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.127944:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00020000:00000004:2.0:1769074539.127952:0:31872:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1487, max_age 1531 00000100:00100000:2.0:1769074539.127987:0:31872:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380288:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074539.128066:0:31872:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074539.134555:0:31872:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:31872:1855007612380288:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074539.134586:0:31872:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074539.134722:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134744:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134751:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134755:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134759:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134763:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134767:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134770:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134775:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134780:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134785:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134788:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134791:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134795:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134798:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134801:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134804:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134808:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134812:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134815:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134820:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134823:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134827:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134830:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134833:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134837:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134843:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134847:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134851:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134855:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134860:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134863:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134868:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134872:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134878:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134882:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134887:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134891:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134896:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134900:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134905:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134909:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134914:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134917:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134921:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134925:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134930:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134933:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134936:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134940:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134943:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134949:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134957:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134961:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134965:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134969:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134972:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134976:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134980:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134984:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134988:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134992:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.134995:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.134998:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135001:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135004:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135007:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135011:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135016:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135020:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135025:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135031:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135035:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135040:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135045:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135048:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135053:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135057:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135063:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135067:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135070:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135074:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135079:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135084:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135090:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135094:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135098:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135102:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135109:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135113:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135118:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135120:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135124:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135127:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135130:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135134:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135138:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135141:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135144:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135147:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135150:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135153:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135156:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135160:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135164:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135168:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135171:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135174:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135177:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135180:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135183:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135186:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135190:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135195:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135199:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135202:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135206:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135210:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135215:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135218:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135222:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135225:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135228:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135233:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200080:2.0:1769074539.135238:0:31872:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd047a32f0 00020000:00000080:2.0:1769074539.135242:0:31872:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096abc00 uarg=0000000000000000 00000080:00200008:2.0:1769074539.135261:0:31872:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:1.0:1769074540.063838:0:998:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 6/6 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1769074540.063882:0:998:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 6/12 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1769074540.063937:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 6 locks of 6 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074540.063949:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f0400/0x68a7113811d7e37d lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf30587fa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.063987:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x1e:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00000080:00200000:1.0:1769074540.064091:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x1e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.064117:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771a00/0x68a7113811d7e353 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf305878a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.064129:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x1c:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00010000:00010000:2.0:1769074540.064270:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 6 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074540.064278:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770c00/0x68a7113811d7e368 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf30587c2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064295:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x1d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064325:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x1d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064339:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030a00/0x68a7113811d7e33e lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf3058752 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064351:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x1b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064371:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x1b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064382:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770000/0x68a7113811d7e314 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf30586e2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064392:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x19:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064410:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x19:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064420:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770400/0x68a7113811d7e2ea lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf3058672 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064435:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064455:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x17:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064479:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0800/0x68a7113811d7e2c0 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf3058602 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064492:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064512:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x15:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064523:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a600/0x68a7113811d7e10e lrc: 2/0,0 mode: PR/PR res: [0x280000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf3057022 expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074540.064533:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074540.064538:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074540.064584:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770c00/0x68a7113811d7e368 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30587c2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064594:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030a00/0x68a7113811d7e33e lrc: 1/0,0 mode: --/PW res: [0x280000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058752 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064601:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770000/0x68a7113811d7e314 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30586e2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064607:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770400/0x68a7113811d7e2ea lrc: 1/0,0 mode: --/PW res: [0x280000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058672 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064614:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0800/0x68a7113811d7e2c0 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058602 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064628:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a600/0x68a7113811d7e10e lrc: 1/0,0 mode: --/PR res: [0x280000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3057022 expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064635:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 6 locks packed 00010000:00010000:2.0:1769074540.064705:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770c00/0x68a7113811d7e368 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30587c2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064731:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1000030a00/0x68a7113811d7e33e lrc: 0/0,0 mode: --/PW res: [0x280000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058752 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064744:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770000/0x68a7113811d7e314 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30586e2 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064755:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100a770400/0x68a7113811d7e2ea lrc: 0/0,0 mode: --/PW res: [0x280000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058672 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064766:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b10097e0800/0x68a7113811d7e2c0 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3058602 expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064777:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b100164a600/0x68a7113811d7e10e lrc: 0/0,0 mode: --/PR res: [0x280000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3057022 expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074540.064790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074540.064792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074540.064794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074540.064820:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612380416:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074540.067674:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074540.067861:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x1c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.068898:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770a00/0x68a7113811d7e329 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf305871a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.068954:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074540.070595:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.070661:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030e00/0x68a7113811d7e2ff lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf30586aa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.070693:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x18:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074540.070740:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x18:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.070764:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770600/0x68a7113811d7e2d5 lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf305863a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.070775:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074540.070794:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x16:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.070805:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164be00/0x68a7113811d7e107 lrc: 2/0,0 mode: PR/PR res: [0x240000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf305701b expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074540.070814:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074540.070819:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074540.070870:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f0400/0x68a7113811d7e37d lrc: 1/0,0 mode: --/PW res: [0x240000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30587fa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070879:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771a00/0x68a7113811d7e353 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305878a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070885:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770a00/0x68a7113811d7e329 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305871a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070892:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030e00/0x68a7113811d7e2ff lrc: 1/0,0 mode: --/PW res: [0x240000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30586aa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070897:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770600/0x68a7113811d7e2d5 lrc: 1/0,0 mode: --/PW res: [0x240000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305863a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070903:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164be00/0x68a7113811d7e107 lrc: 1/0,0 mode: --/PR res: [0x240000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305701b expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070927:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 6 locks packed 00010000:00010000:1.0:1769074540.070962:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b10087f0400/0x68a7113811d7e37d lrc: 0/0,0 mode: --/PW res: [0x240000400:0x186:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30587fa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070987:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a771a00/0x68a7113811d7e353 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x185:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305878a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.070999:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770a00/0x68a7113811d7e329 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x184:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305871a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.071011:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1000030e00/0x68a7113811d7e2ff lrc: 0/0,0 mode: --/PW res: [0x240000400:0x183:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf30586aa expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.071046:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100a770600/0x68a7113811d7e2d5 lrc: 0/0,0 mode: --/PW res: [0x240000400:0x182:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305863a expref: -99 pid: 27002 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.071058:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b100164be00/0x68a7113811d7e107 lrc: 0/0,0 mode: --/PR res: [0x240000400:0xc2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf305701b expref: -99 pid: 19613 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074540.071074:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074540.071076:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074540.071078:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074540.071094:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074540.071121:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612380544:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074540.078577:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612380416:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074540.082869:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612380544:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074540.192184:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612380672:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074540.193620:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612380800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074540.205983:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612380672:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074540.206137:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612380800:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074541.087972:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074542.111913:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074543.139099:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074543.622851:0:32122:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:2.0:1769074544.161453:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074545.184402:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074545.312027:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612380928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.312113:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381184:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.312138:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074545.312144:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.312167:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381312:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.325624:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612380928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.333481:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.333504:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381312:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074545.333516:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612381184:192.168.204.102@tcp:400:kworker.0 00000001:02000400:0.0:1769074545.572038:0:32276:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1769074546.218300:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074547.239804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1769074547.799611:0:32314:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074547.799632:0:32314:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074547.799684:0:32314:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074547.799708:0:32314:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.799727:0:32314:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074547.799980:0:32314:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074547.799990:0:32314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074547.799994:0:32314:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:3.0:1769074547.800003:0:32314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141cdbcb8 00010000:00010000:3.0:1769074547.800008:0:32314:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.800018:0:32314:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074547.802090:0:32314:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074547.802101:0:32314:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074547.802126:0:32314:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074547.802145:0:32314:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.802160:0:32314:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074547.802167:0:32314:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074547.802176:0:32314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074547.802180:0:32314:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:3.0:1769074547.802188:0:32314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141cdbcb8 00010000:00010000:3.0:1769074547.802191:0:32314:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.802201:0:32314:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074547.805556:0:32314:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074547.805599:0:32314:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074547.805621:0:32314:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.805663:0:32314:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074547.805673:0:32314:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074547.805683:0:32314:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074547.805687:0:32314:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:3.0:1769074547.805696:0:32314:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141cdbbb0 00010000:00010000:3.0:1769074547.805700:0:32314:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074547.805708:0:32314:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074547.805728:0:32314:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074547.805752:0:32314:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074547.805923:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.805945:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00800000:00000004:3.0:1769074547.805954:0:32314:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1532, max_age 1540 00000100:00100000:3.0:1769074547.806011:0:32314:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381440:192.168.204.102@tcp:41:lfs.0 00000100:00100000:3.0:1769074547.806130:0:32314:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2d3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074547.809862:0:32314:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381440:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074547.809893:0:32314:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:3.0:1769074547.809974:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.809992:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.809999:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810004:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810008:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810011:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810014:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810018:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810181:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810189:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810196:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810200:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810204:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810207:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810211:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810214:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810218:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810222:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810226:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810229:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810233:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810239:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810244:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810248:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810253:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810257:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810261:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810265:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810269:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810273:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810277:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810281:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810286:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810291:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810298:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810302:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810310:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810315:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810322:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810326:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810343:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810347:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810352:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810356:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810361:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810365:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810371:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810375:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810380:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810383:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810387:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810390:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810393:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810397:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810400:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810404:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810408:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810410:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810414:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810417:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810420:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810424:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810427:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810429:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810433:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810436:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810439:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810441:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810445:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810448:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810451:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810456:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810461:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810464:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810476:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810481:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810496:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810502:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810508:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810512:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810517:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810523:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810527:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810531:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810547:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810552:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810557:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810561:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810565:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810569:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810573:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810576:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810580:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810584:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810588:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810591:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810596:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810599:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810603:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810606:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810609:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810613:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810617:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810620:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810624:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810627:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810631:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810633:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810839:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810847:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810854:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810858:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810862:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810866:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810870:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810873:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810877:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810880:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810884:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810887:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810891:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810894:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810898:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810901:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810905:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810908:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810912:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00800000:00000080:3.0:1769074547.810915:0:32314:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.810919:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.810925:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00020000:00000004:3.0:1769074547.810934:0:32314:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1532, max_age 1540 00000100:00100000:3.0:1769074547.810977:0:32314:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381568:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074547.811850:0:32314:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2d3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074547.818208:0:32314:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381568:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074547.818240:0:32314:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074547.818318:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.818345:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00020000:00000004:3.0:1769074547.818353:0:32314:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1532, max_age 1540 00000100:00100000:3.0:1769074547.818389:0:32314:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381696:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074547.818472:0:32314:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2d3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074547.828212:0:32314:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32314:1855007612381696:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074547.828251:0:32314:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074547.828343:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828364:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828372:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828378:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828382:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828386:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828391:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828395:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828400:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828403:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828408:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828412:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828417:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828422:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828426:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828430:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828434:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828438:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828442:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828446:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828450:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828454:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828458:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828461:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828465:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828469:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828473:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828476:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828481:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828487:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828492:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828497:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828501:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828506:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828510:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828513:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828516:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828520:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828523:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828526:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828530:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828544:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828550:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828554:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828559:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828564:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828569:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828574:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828579:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828582:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828586:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828590:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828594:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828600:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828604:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828608:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828612:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828615:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828620:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828623:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828630:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828634:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828679:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828684:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828688:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828692:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828696:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828699:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828702:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828706:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828709:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828713:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828717:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828720:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828724:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828727:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828731:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828735:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828741:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828745:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828750:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828754:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828758:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828762:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828768:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828772:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828776:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828780:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828786:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828792:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828797:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828800:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828805:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828810:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828816:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828819:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828824:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828827:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828830:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828833:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828838:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828843:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828848:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828853:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828857:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828862:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828867:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828871:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828875:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828879:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828884:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828889:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828893:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828897:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828901:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828905:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828909:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828913:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828918:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828924:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828929:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828933:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828939:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828942:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200080:3.0:1769074547.828946:0:32314:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffda63f4b70 00020000:00000080:3.0:1769074547.828951:0:32314:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35000 uarg=0000000000000000 00000080:00200008:3.0:1769074547.828971:0:32314:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:0.0:1769074547.890483:0:32330:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074547.890523:0:32330:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074547.890559:0:32330:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074547.890580:0:32330:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074547.890590:0:32330:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074547.890601:0:32330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074547.890606:0:32330:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:0.0:1769074547.890615:0:32330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa91414fbb58 00010000:00010000:0.0:1769074547.890619:0:32330:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074547.890629:0:32330:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074547.890695:0:32330:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074547.890711:0:32330:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:0.0:1769074547.890716:0:32330:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:0.0:1769074547.890733:0:32330:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074547.890801:0:32330:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32330 initiator: MDT0 00010000:00010000:0.0:1769074547.890811:0:32330:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32330 initiator: MDT0 00010000:00010000:0.0:1769074547.890823:0:32330:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32330 initiator: MDT0 00000100:00100000:0.0:1769074547.890859:0:32330:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:32330:1855007612381824:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:0.0:1769074547.890989:0:32330:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2240 going to sleep for 11 seconds 00000100:00100000:2.0:1769074547.917490:0:32330:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:32330:1855007612381824:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:2.0:1769074547.917527:0:32330:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917567:0:32330:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917583:0:32330:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917599:0:32330:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917612:0:32330:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917626:0:32330:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917635:0:32330:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917672:0:32330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074547.917685:0:32330:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006025f80 x1855007612381824/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074563 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074547.917699:0:32330:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917711:0:32330:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/263 (0 0) 00000080:00200000:2.0:1769074547.917718:0:32330:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074547.917747:0:32330:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 11776 00000080:00200000:2.0:1769074547.917751:0:32330:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x107:0x0](ffff8b100485cb08): rc = 0 00000080:00010000:2.0:1769074547.917758:0:32330:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) for lock 0x68a7113811d80454 00000080:00010000:2.0:1769074547.917775:0:32330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) 00000080:00010000:2.0:1769074547.917780:0:32330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80454 from it ffffaa91414fbb20 00010000:00010000:2.0:1769074547.917784:0:32330:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917794:0:32330:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074547.917838:0:32330:0:(file.c:6252:ll_getattr_dentry()) START file d6a.replay-single:[0x200001b71:0x107:0x0](ffff8b100485cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074547.917845:0:32330:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08),name=d6a.replay-single 00000002:00010000:2.0:1769074547.917874:0:32330:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x107:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074547.917884:0:32330:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917896:0:32330:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074547.917902:0:32330:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074547.917908:0:32330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) 00000080:00010000:2.0:1769074547.917912:0:32330:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffff8b100485cb08) for lock 0x68a7113811d80454 00000080:00010000:2.0:1769074547.917916:0:32330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80454 from it ffffaa91414fbcb8 00010000:00010000:2.0:1769074547.917919:0:32330:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074547.917926:0:32330:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074547.939529:0:32333:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074547.939588:0:32333:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074547.939614:0:32333:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.939636:0:32333:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074547.939854:0:32333:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074547.939865:0:32333:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074547.939870:0:32333:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:1.0:1769074547.939880:0:32333:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa91414fbb58 00010000:00010000:1.0:1769074547.939887:0:32333:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.939898:0:32333:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074547.939921:0:32333:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074547.939931:0:32333:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6a.replay-single, flags=24 00000080:00200000:1.0:1769074547.945900:0:32333:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffff8b100485cb08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074547.945950:0:32333:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffff8b100485cb08), flags=8 00000080:00200000:1.0:1769074547.945958:0:32333:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffff8b100485cb08), intent=getattr 00000002:00010000:1.0:1769074547.945992:0:32333:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f6a.replay-single,[0x0:0x0:0x0]) in obj [0x200001b71:0x107:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074547.946080:0:32333:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x107:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32333 initiator: MDT0 00010000:00010000:1.0:1769074547.946093:0:32333:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x107:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32333 initiator: MDT0 00010000:00010000:1.0:1769074547.946105:0:32333:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x107:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32333 initiator: MDT0 00000100:00100000:1.0:1769074547.946142:0:32333:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:32333:1855007612381952:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:1.0:1769074547.946273:0:32333:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364ccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074547.950738:0:32333:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:32333:1855007612381952:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:1.0:1769074547.950777:0:32333:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x107:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950807:0:32333:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950824:0:32333:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950853:0:32333:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950866:0:32333:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950880:0:32333:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950890:0:32333:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950898:0:32333:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074547.950912:0:32333:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006136680 x1855007612381952/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074563 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074547.950928:0:32333:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.950940:0:32333:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/265 (0 0) 00000080:00200000:1.0:1769074547.950948:0:32333:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x107:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074547.950977:0:32333:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x109:0x0], updating i_size 0 00000080:00200000:1.0:1769074547.950980:0:32333:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x109:0x0](ffff8b100486ec08): rc = 0 00000080:00200000:1.0:1769074547.951001:0:32333:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x109:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1769074547.951010:0:32333:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x109:0x0](ffff8b100486ec08) for lock 0x68a7113811d8045b 00000080:00010000:1.0:1769074547.951025:0:32333:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x109:0x0](ffff8b100486ec08) 00000080:00010000:1.0:1769074547.951030:0:32333:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8045b from it ffffaa91414fbb20 00010000:00010000:1.0:1769074547.951034:0:32333:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.951044:0:32333:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074547.951101:0:32333:0:(file.c:6252:ll_getattr_dentry()) START file f6a.replay-single:[0x200001b71:0x109:0x0](ffff8b100486ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074547.951108:0:32333:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x109:0x0](ffff8b100486ec08),name=f6a.replay-single 00000002:00010000:1.0:1769074547.951137:0:32333:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x109:0x0]) in obj [0x200001b71:0x107:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074547.951147:0:32333:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.951163:0:32333:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074547.951171:0:32333:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x107:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074547.951176:0:32333:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x109:0x0](ffff8b100486ec08) 00000080:00010000:1.0:1769074547.951182:0:32333:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x109:0x0](ffff8b100486ec08) for lock 0x68a7113811d8045b 00000080:00010000:1.0:1769074547.951187:0:32333:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8045b from it ffffaa91414fbcb8 00010000:00010000:1.0:1769074547.951190:0:32333:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074547.951197:0:32333:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074548.256358:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074549.281135:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074550.304948:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074550.433969:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074550.434094:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074550.434131:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074550.434137:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612382080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074550.434170:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074550.437732:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612382080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074550.437765:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074550.441378:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074550.446965:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612382464:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074551.328938:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074552.362324:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074553.375833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074554.401049:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074555.427017:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074555.555768:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612382592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.555869:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.555906:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074555.555914:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.555940:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382976:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.586880:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.586921:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.586932:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612382976:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074555.586941:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612382592:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074556.447754:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074557.474725:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074557.477610:0:32899:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 6b: |X| rmdir ====================== 04:35:56 (1769074556) 00000080:00200000:1.0:1769074557.669728:0:32908:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074557.679484:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074557.679512:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074557.679528:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074557.679535:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074557.679542:0:32908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074557.679546:0:32908:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:1.0:1769074557.679552:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067b58 00010000:00010000:1.0:1769074557.679556:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074557.679565:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074557.679581:0:32908:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074557.679601:0:32908:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=13 00000080:00200000:1.0:1769074557.679606:0:32908:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:1.0:1769074557.679619:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6b.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074557.679693:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32908 initiator: MDT0 00010000:00010000:1.0:1769074557.679702:0:32908:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32908 initiator: MDT0 00010000:00010000:1.0:1769074557.679711:0:32908:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32908 initiator: MDT0 00000100:00100000:1.0:1769074557.679738:0:32908:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32908:1855007612383104:192.168.204.102@tcp:101:lfs.0 00000100:00100000:1.0:1769074557.679827:0:32908:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10056e5240 going to sleep for 11 seconds 00000100:00100000:2.0:1769074557.685466:0:32908:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32908:1855007612383104:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074557.685492:0:32908:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685505:0:32908:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685512:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685518:0:32908:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685526:0:32908:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685533:0:32908:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8c00 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074557.685538:0:32908:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685545:0:32908:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685553:0:32908:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685560:0:32908:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074557.685567:0:32908:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 32908 initiator: MDT0 00010000:00010000:2.0:1769074557.685574:0:32908:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80462 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 32908 initiator: MDT0 00000002:00100000:2.0:1769074557.685590:0:32908:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1006136680 x1855007612383104/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074573 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074557.685606:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074557.685634:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685674:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074557.685685:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067940 00010000:00010000:2.0:1769074557.685688:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685697:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.685780:0:32908:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074557.685794:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074557.685801:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685810:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.685816:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074557.685821:0:32908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074557.685824:0:32908:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074557.685829:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067be8 00010000:00010000:2.0:1769074557.685832:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685844:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.685853:0:32908:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074557.685861:0:32908:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=16392 00000080:00200008:2.0:1769074557.685884:0:32908:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074557.685889:0:32908:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074557.685896:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074557.685901:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685909:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.685915:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074557.685919:0:32908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074557.685921:0:32908:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074557.685936:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067cb8 00010000:00010000:2.0:1769074557.685939:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.685946:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074557.686636:0:32908:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074557.686666:0:32908:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074557.686676:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074557.686685:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.686695:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.686702:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074557.686706:0:32908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074557.686709:0:32908:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074557.686713:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067cb8 00010000:00010000:2.0:1769074557.686716:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.686724:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.686844:0:32908:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074557.686853:0:32908:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074557.686859:0:32908:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.686868:0:32908:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.686874:0:32908:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074557.686878:0:32908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074557.686881:0:32908:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8044d 00000080:00010000:2.0:1769074557.686884:0:32908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8044d from it ffffaa9141067bb0 00010000:00010000:2.0:1769074557.686887:0:32908:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074557.686895:0:32908:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074557.686904:0:32908:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074557.686911:0:32908:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00000080:00200080:2.0:1769074557.686929:0:32908:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c03066f0 arg=7ffe14750810 00000080:00200000:2.0:1769074557.686941:0:32908:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) name=d6b.replay-single stripe_offset=0 stripe_count=1, hash_type=0 00010000:00010000:3.0:1769074557.693888:0:32908:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074557.694758:0:32908:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074557.694771:0:32908:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074557.694774:0:32908:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d8044d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3062fde expref: -99 pid: 31872 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074557.694806:0:32908:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074557.694819:0:32908:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037ded80 x1855007612383360/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074557.694843:0:32908:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32908:1855007612383360:192.168.204.102@tcp:36:lfs.0 00000100:00100000:3.0:1769074557.694945:0:32908:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0f00 going to sleep for 11 seconds 00000100:00100000:3.0:1769074557.705040:0:32908:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32908:1855007612383360:192.168.204.102@tcp:36:lfs.0 00000080:00200000:3.0:1769074557.705094:0:32908:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88) 00000080:00200000:3.0:1769074557.705102:0:32908:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10a:0x0], updating i_size 512 00000080:00200008:3.0:1769074557.705142:0:32908:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00010000:00010000:2.0:1769074558.496714:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074558.861766:0:32920:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074558.861781:0:32920:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.861804:0:32920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.861869:0:32920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 32920 initiator: MDT0 00010000:00010000:2.0:1769074558.861881:0:32920:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32920 initiator: MDT0 00010000:00010000:2.0:1769074558.861891:0:32920:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32920 initiator: MDT0 00000100:00100000:2.0:1769074558.861920:0:32920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383488:192.168.204.102@tcp:101:lfs.0 00000100:00100000:2.0:1769074558.862009:0:32920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074558.876153:0:32920:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383488:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074558.876178:0:32920:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876189:0:32920:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876202:0:32920:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876213:0:32920:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876224:0:32920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876232:0:32920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876239:0:32920:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074558.876251:0:32920:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d58e00 x1855007612383488/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074574 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074558.876264:0:32920:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876277:0:32920:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074558.876283:0:32920:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074558.876312:0:32920:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074558.876318:0:32920:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:2.0:1769074558.876323:0:32920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074558.876326:0:32920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa9141067cb8 00010000:00010000:2.0:1769074558.876329:0:32920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.876338:0:32920:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074558.877157:0:32920:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074558.877164:0:32920:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.877179:0:32920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.877188:0:32920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.877199:0:32920:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.877205:0:32920:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074558.877210:0:32920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074558.877213:0:32920:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:2.0:1769074558.877217:0:32920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa9141067cb8 00010000:00010000:2.0:1769074558.877220:0:32920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.877226:0:32920:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.877329:0:32920:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.877337:0:32920:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.877344:0:32920:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074558.879710:0:32920:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074558.879727:0:32920:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074558.879736:0:32920:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074558.879740:0:32920:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:3.0:1769074558.879748:0:32920:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa9141067bb0 00010000:00010000:3.0:1769074558.879752:0:32920:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074558.879759:0:32920:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074558.879776:0:32920:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074558.879786:0:32920:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074558.879916:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.879931:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00800000:00000004:3.0:1769074558.879938:0:32920:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1541, max_age 1551 00000100:00100000:3.0:1769074558.879981:0:32920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383616:192.168.204.102@tcp:41:lfs.0 00000100:00100000:3.0:1769074558.880063:0:32920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0f00 going to sleep for 11 seconds 00000100:00100000:3.0:1769074558.882617:0:32920:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383616:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074558.882689:0:32920:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:3.0:1769074558.882760:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882776:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882784:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882788:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882793:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882798:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882803:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882807:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882812:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882817:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882821:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882824:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882827:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882830:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882833:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882836:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882840:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882843:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882847:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882851:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882855:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882859:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882863:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882866:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882869:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882872:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882876:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882879:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882882:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882886:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882889:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882893:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882896:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882899:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882904:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882919:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882925:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882929:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882933:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882936:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882940:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882944:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882948:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882952:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882956:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882960:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.882964:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.882968:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883235:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883244:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883249:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883253:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883257:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883260:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883264:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883268:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883272:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883275:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883279:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883283:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883287:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883291:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883295:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883299:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883303:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883307:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883311:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883315:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883319:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883322:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883326:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883330:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883334:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883338:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883344:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883358:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883364:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883368:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883372:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883376:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883380:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883383:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883388:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883391:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883395:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883398:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883401:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883404:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883408:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883411:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883415:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883418:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883422:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883425:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883428:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883431:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883435:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883438:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883441:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883444:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883449:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883452:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883456:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883460:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883464:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883467:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883471:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883474:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883478:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883482:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883486:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883490:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883494:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883498:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883502:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883506:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883509:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883513:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883516:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883519:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883523:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883527:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883530:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883534:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883538:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883541:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883544:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00800000:00000080:3.0:1769074558.883547:0:32920:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00000080:00200080:3.0:1769074558.883551:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:3.0:1769074558.883557:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f36000 uarg=0000000000000000 00020000:00000004:3.0:1769074558.883565:0:32920:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1541, max_age 1551 00000100:00100000:3.0:1769074558.883605:0:32920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383744:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074558.887134:0:32920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0f00 going to sleep for 11 seconds 00000100:00100000:1.0:1769074558.901728:0:32920:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383744:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074558.901759:0:32920:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074558.901832:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.901850:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00020000:00000004:1.0:1769074558.901858:0:32920:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1541, max_age 1551 00000100:00100000:1.0:1769074558.901895:0:32920:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383872:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074558.901978:0:32920:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10056e5000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074558.905796:0:32920:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:32920:1855007612383872:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074558.905825:0:32920:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074558.905902:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905921:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905928:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905932:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905936:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905940:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905944:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905947:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905951:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905954:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905957:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905960:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905964:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905967:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905971:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905974:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905978:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905982:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905986:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905989:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.905993:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.905996:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906000:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906003:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906007:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906011:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906016:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906020:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906024:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906027:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906031:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906035:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906039:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906043:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906047:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906050:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906055:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906059:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906063:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906067:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906071:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906075:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906078:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906081:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906085:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906088:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906092:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906096:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906100:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906103:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906107:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906111:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906114:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906118:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906122:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906125:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906129:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906132:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906135:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906138:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906142:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906145:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906148:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906151:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906156:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906158:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906162:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906165:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906168:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906172:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906176:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906178:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906182:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906185:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906189:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906193:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906197:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906199:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906203:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906206:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906209:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906212:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906215:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906218:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906221:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906225:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906229:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906232:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906236:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906239:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906243:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906246:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906250:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906253:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906257:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906261:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906265:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906268:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906272:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906275:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906279:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906283:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906287:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906291:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906295:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906299:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906302:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906306:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906310:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906313:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906316:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906319:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906323:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906326:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906330:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906334:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906337:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.906341:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.906346:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.907021:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.907035:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.907040:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.907044:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.907047:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200080:1.0:1769074558.907051:0:32920:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff73661500 00020000:00000080:1.0:1769074558.907055:0:32920:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00000080:00200008:1.0:1769074558.907068:0:32920:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074558.989665:0:32927:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.989705:0:32927:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.989723:0:32927:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989748:0:32927:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989756:0:32927:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074558.989764:0:32927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074558.989767:0:32927:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:2.0:1769074558.989774:0:32927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa914156fbd0 00010000:00010000:2.0:1769074558.989777:0:32927:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989785:0:32927:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989799:0:32927:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074558.989814:0:32927:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074558.989817:0:32927:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.989822:0:32927:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.989827:0:32927:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989835:0:32927:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989840:0:32927:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074558.989844:0:32927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074558.989847:0:32927:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:2.0:1769074558.989850:0:32927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa914156fcd0 00010000:00010000:2.0:1769074558.989853:0:32927:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989860:0:32927:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989867:0:32927:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074558.989873:0:32927:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074558.989879:0:32927:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074558.989884:0:32927:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989891:0:32927:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989896:0:32927:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074558.989899:0:32927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074558.989901:0:32927:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80469 00000080:00010000:2.0:1769074558.989904:0:32927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80469 from it ffffaa914156fdb0 00010000:00010000:2.0:1769074558.989906:0:32927:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.989912:0:32927:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074558.989918:0:32927:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074558.989922:0:32927:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074558.989926:0:32927:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074558.989947:0:32927:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.990000:0:32927:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074558.990006:0:32927:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074558.990009:0:32927:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698e00/0x68a7113811d80469 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063055 expref: -99 pid: 32920 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074558.990033:0:32927:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074558.990046:0:32927:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58e00 x1855007612384128/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074558.990069:0:32927:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:32927:1855007612384128:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074558.990157:0:32927:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074559.005214:0:32927:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:32927:1855007612384128:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074559.005267:0:32927:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffff8b100486aa08) 00000080:00200000:2.0:1769074559.005304:0:32927:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:0.0:1769074559.048993:0:32930:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074559.049020:0:32930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074559.049086:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 32930 initiator: MDT0 00010000:00010000:0.0:1769074559.049097:0:32930:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32930 initiator: MDT0 00010000:00010000:0.0:1769074559.049106:0:32930:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32930 initiator: MDT0 00000100:00100000:0.0:1769074559.049129:0:32930:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384256:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074559.049216:0:32930:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442000 going to sleep for 11 seconds 00000100:00100000:0.0:1769074559.061759:0:32930:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384256:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074559.061790:0:32930:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061808:0:32930:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061820:0:32930:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061830:0:32930:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061842:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061849:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061855:0:32930:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074559.061866:0:32930:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024380 x1855007612384256/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074575 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074559.061880:0:32930:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061891:0:32930:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074559.061898:0:32930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074559.061924:0:32930:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:0.0:1769074559.061930:0:32930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80470 00000080:00010000:0.0:1769074559.061935:0:32930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074559.061939:0:32930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80470 from it ffffaa9141d93b58 00010000:00010000:0.0:1769074559.061942:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.061950:0:32930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.061973:0:32930:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074559.061993:0:32930:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:0.0:1769074559.061998:0:32930:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:0.0:1769074559.062018:0:32930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074559.062057:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32930 initiator: MDT0 00010000:00010000:0.0:1769074559.062064:0:32930:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32930 initiator: MDT0 00010000:00010000:0.0:1769074559.062073:0:32930:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32930 initiator: MDT0 00000100:00100000:0.0:1769074559.062092:0:32930:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384384:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074559.062161:0:32930:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442000 going to sleep for 11 seconds 00000100:00100000:0.0:1769074559.073404:0:32930:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384384:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074559.073432:0:32930:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073455:0:32930:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073474:0:32930:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073487:0:32930:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073502:0:32930:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073514:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073522:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073529:0:32930:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074559.073541:0:32930:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024380 x1855007612384384/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074575 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074559.073552:0:32930:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073561:0:32930:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/267 (0 0) 00000080:00200000:0.0:1769074559.073567:0:32930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074559.073586:0:32930:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10b:0x0], updating i_size 0 00000080:00200000:0.0:1769074559.073589:0:32930:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10b:0x0](ffff8b100486aa08): rc = 0 00000080:00200000:0.0:1769074559.073606:0:32930:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x10b:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074559.073611:0:32930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10b:0x0](ffff8b100486aa08) for lock 0x68a7113811d80477 00000080:00010000:0.0:1769074559.073622:0:32930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10b:0x0](ffff8b100486aa08) 00000080:00010000:0.0:1769074559.073627:0:32930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80477 from it ffffaa9141d93b20 00010000:00010000:0.0:1769074559.073630:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073669:0:32930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074559.073708:0:32930:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x10b:0x0](ffff8b100486aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074559.073715:0:32930:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffff8b100486aa08),name=fsa-oleg402-client.virtnet 00000002:00010000:0.0:1769074559.073735:0:32930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074559.073743:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073753:0:32930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073759:0:32930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074559.073763:0:32930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10b:0x0](ffff8b100486aa08) 00000080:00010000:0.0:1769074559.073766:0:32930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10b:0x0](ffff8b100486aa08) for lock 0x68a7113811d80477 00000080:00010000:0.0:1769074559.073770:0:32930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80477 from it ffffaa9141d93cb8 00010000:00010000:0.0:1769074559.073773:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073779:0:32930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073837:0:32930:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074559.073844:0:32930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074559.073851:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073858:0:32930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073864:0:32930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074559.073868:0:32930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074559.073870:0:32930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80470 00000080:00010000:0.0:1769074559.073875:0:32930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80470 from it ffffaa9141d93c10 00010000:00010000:0.0:1769074559.073877:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073883:0:32930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073890:0:32930:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074559.073899:0:32930:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:0.0:1769074559.073903:0:32930:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074559.073908:0:32930:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074559.073913:0:32930:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073919:0:32930:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073924:0:32930:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074559.073928:0:32930:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074559.073930:0:32930:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80470 00000080:00010000:0.0:1769074559.073933:0:32930:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80470 from it ffffaa9141d93d68 00010000:00010000:0.0:1769074559.073936:0:32930:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073942:0:32930:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.073948:0:32930:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074559.073954:0:32930:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074559.073974:0:32930:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.073994:0:32930:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074559.074003:0:32930:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x10b:0x0]: losing layout lock 00010000:00010000:0.0:1769074559.074034:0:32930:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.074041:0:32930:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.074046:0:32930:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074559.074048:0:32930:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d80477 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306308d expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074559.074066:0:32930:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d80470 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063071 expref: -99 pid: 32930 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074559.074082:0:32930:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074559.074089:0:32930:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006024380 x1855007612384640/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074559.074123:0:32930:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384640:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074559.074195:0:32930:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442000 going to sleep for 11 seconds 00000100:00100000:0.0:1769074559.097417:0:32930:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32930:1855007612384640:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074559.097495:0:32930:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x10b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00000080:00200000:0.0:1769074559.097506:0:32930:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x10b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074559.097514:0:32930:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffff8b100486aa08) 00010000:00010000:2.0:1769074559.521674:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074560.544950:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074560.671971:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612384768:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074560.672100:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612385024:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074560.672265:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612384896:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074560.672360:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612385152:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074560.675480:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612385024:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074560.675804:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612385152:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074560.685197:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612384896:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074560.685778:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612384768:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074561.567983:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074562.515382:0:32960:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1769074562.568496:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074562.568539:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.568631:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.568662:0:32961:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.568674:0:32961:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00000100:00100000:2.0:1769074562.568708:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385280:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074562.568823:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074562.579176:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385280:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074562.579214:0:32961:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579230:0:32961:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579245:0:32961:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579256:0:32961:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579270:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579279:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579301:0:32961:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074562.579313:0:32961:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5ad80 x1855007612385280/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074578 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074562.579327:0:32961:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579341:0:32961:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074562.579349:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074562.579386:0:32961:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074562.579394:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8047e 00000080:00010000:2.0:1769074562.579399:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074562.579403:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8047e from it ffffaa9141d03b58 00010000:00010000:2.0:1769074562.579406:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.579415:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.579802:0:32961:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074562.579835:0:32961:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074562.579841:0:32961:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074562.579871:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6b.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.580560:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.580579:0:32961:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.580589:0:32961:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00000100:00100000:2.0:1769074562.580623:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385408:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074562.580939:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00010000:00010000:2.0:1769074562.593581:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074562.594614:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385408:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074562.594682:0:32961:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594711:0:32961:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594719:0:32961:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594734:0:32961:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594748:0:32961:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594763:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594772:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594780:0:32961:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074562.594793:0:32961:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5ad80 x1855007612385408/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074578 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074562.594806:0:32961:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594817:0:32961:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/266 (0 0) 00000080:00200000:2.0:1769074562.594824:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074562.594854:0:32961:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10a:0x0], updating i_size 11776 00000080:00200000:2.0:1769074562.594858:0:32961:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10a:0x0](ffff8b100485db88): rc = 0 00000080:00010000:2.0:1769074562.594865:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) for lock 0x68a7113811d80485 00000080:00010000:2.0:1769074562.594880:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) 00000080:00010000:2.0:1769074562.594885:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80485 from it ffffaa9141d03b20 00010000:00010000:2.0:1769074562.594888:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.594897:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074562.594972:0:32961:0:(file.c:6252:ll_getattr_dentry()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffff8b100485db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074562.594980:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88),name=d6b.replay-single 00000002:00010000:2.0:1769074562.595007:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.595017:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.595035:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.595041:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074562.595046:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) 00000080:00010000:2.0:1769074562.595050:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) for lock 0x68a7113811d80485 00000080:00010000:2.0:1769074562.595054:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80485 from it ffffaa9141d03cb8 00010000:00010000:2.0:1769074562.595057:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.595063:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.595148:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074562.595157:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.595164:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.595172:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.595178:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074562.595182:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074562.595185:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8047e 00000080:00010000:2.0:1769074562.595188:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8047e from it ffffaa9141d03bf8 00010000:00010000:2.0:1769074562.595192:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.595203:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.595210:0:32961:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074562.595217:0:32961:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=266 00000080:00200000:2.0:1769074562.595224:0:32961:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074562.595234:0:32961:0:(file.c:1001:ll_file_open()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffff8b100485db88), flags 704400 00000002:00010000:2.0:1769074562.595249:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:2.0:1769074562.595318:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.595326:0:32961:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00000020:00100000:2.0:1769074562.595338:0:32961:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074562.595347:0:32961:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5ad80 x1855007612385664/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074562.595356:0:32961:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00000100:00100000:2.0:1769074562.595382:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385664:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074562.595952:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074562.607701:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385664:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074562.607736:0:32961:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607754:0:32961:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607766:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607775:0:32961:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607785:0:32961:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607791:0:32961:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699c00 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074562.607797:0:32961:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607803:0:32961:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607811:0:32961:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607818:0:32961:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074562.607825:0:32961:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 32961 initiator: MDT0 00010000:00010000:2.0:1769074562.607831:0:32961:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d8048c lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 32961 initiator: MDT0 00000002:00100000:2.0:1769074562.607868:0:32961:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1005d5ad80 x1855007612385664/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074578 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074562.607884:0:32961:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5ad80 x1855007612385664/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074578 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074562.607899:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074562.608088:0:32961:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10a:0x0], updating i_size 11776 00000002:00100000:2.0:1769074562.608120:0:32961:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5ad80 x1855007612385664/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074578 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1769074562.608226:0:32961:0:(file.c:6252:ll_getattr_dentry()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffff8b100485db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074562.608235:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88),name=d6b.replay-single 00000002:00010000:2.0:1769074562.608269:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.608298:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.608312:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.608320:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074562.608328:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) 00000080:00010000:2.0:1769074562.608332:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffff8b100485db88) for lock 0x68a7113811d80485 00000080:00010000:2.0:1769074562.608341:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80485 from it ffffaa9141d03cd8 00010000:00010000:2.0:1769074562.608344:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.608352:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.611469:0:32961:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88) pos/size0/11776 32bit_api 0 00000002:00010000:2.0:1769074562.611520:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200001b71:0x10a:0x0], intent: readdir flags 0 00010000:00010000:2.0:1769074562.611543:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.611560:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074562.612258:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385792:192.168.204.102@tcp:37:rm.0 00000100:00100000:2.0:1769074562.612405:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074562.623382:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612385792:192.168.204.102@tcp:37:rm.0 00010000:00010000:2.0:1769074562.624167:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.624187:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.624259:0:32961:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88) pos/size18446744073709551614/11776 32bit_api 0 00000080:00200008:2.0:1769074562.624380:0:32961:0:(file.c:424:ll_file_release()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffff8b100485db88), flags 704000 00000002:00100000:2.0:1769074562.624471:0:32961:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005d5ad80 x1855007612385664/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074578 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074562.624487:0:32961:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074562.624494:0:32961:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59f80 x1855007612386048/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074562.624524:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612386048:192.168.204.102@tcp:35:rm.0 00000100:00100000:2.0:1769074562.624623:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074562.628758:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612386048:192.168.204.102@tcp:35:rm.0 00000002:00100000:2.0:1769074562.628808:0:32961:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1005d5ad80 x1855007612385664/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074578 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074562.630177:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074562.630216:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.630241:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630262:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.630271:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074562.630281:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074562.630295:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8047e 00000080:00010000:2.0:1769074562.630304:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8047e from it ffffaa9141d03c10 00010000:00010000:2.0:1769074562.630308:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630317:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.630335:0:32961:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074562.630352:0:32961:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=0 00000080:00200000:2.0:1769074562.630359:0:32961:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074562.630366:0:32961:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074562.630371:0:32961:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630379:0:32961:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.630385:0:32961:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074562.630389:0:32961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074562.630391:0:32961:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8047e 00000080:00010000:2.0:1769074562.630395:0:32961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8047e from it ffffaa9141d03d88 00010000:00010000:2.0:1769074562.630399:0:32961:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630405:0:32961:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074562.630412:0:32961:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074562.630418:0:32961:0:(namei.c:2264:ll_rmdir()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074562.630443:0:32961:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630473:0:32961:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630591:0:32961:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630601:0:32961:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630607:0:32961:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074562.630610:0:32961:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d80485 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30630d3 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074562.630629:0:32961:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d8047e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30630b7 expref: -99 pid: 32961 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074562.631037:0:32961:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074562.631054:0:32961:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59f80 x1855007612386304/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074562.631093:0:32961:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612386304:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074562.631216:0:32961:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074562.642025:0:32961:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:32961:1855007612386304:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074562.642096:0:32961:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffff8b100485db88) 00010000:00010000:2.0:1769074563.617441:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074564.642060:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074565.664472:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074565.792170:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074565.792418:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074565.792463:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1769074565.792470:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612386432:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074565.792506:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386688:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074565.801036:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386688:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074565.801065:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386816:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074566.688411:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074567.712269:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074568.735848:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074569.760694:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074570.786607:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074570.913061:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612386944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074570.913347:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074570.913356:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074570.914293:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074570.914359:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074570.914366:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387328:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074570.942118:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387328:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074570.942161:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387200:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074571.808209:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074572.831912:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074573.856910:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074574.879958:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074575.904702:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074576.035630:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612387456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074576.040236:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074576.040273:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387584:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074576.041174:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612387840:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074576.041242:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074576.041250:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612387712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074576.055919:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612387712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074576.055945:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612387840:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074576.928967:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074577.952990:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074578.976102:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074580.000070:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074581.023798:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769074581.152032:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074565/real 1769074565] req@ffff8b10037ddf80 x1855007612386432/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074581 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074581.152036:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:2.0:1769074581.152050:0:16087:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b101127b800: disconnect after 23s idle 00000100:00100000:3.0:1769074581.152093:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037ddf80 x1855007612386432/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074581 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074581.152096:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:2.0:1769074581.152110:0:16087:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b101127b800: disconnect after 23s idle 00000100:00080000:3.0:1769074581.152120:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1769074581.152128:0:2367:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1769074581.152133:0:2367:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1769074581.152145:0:2367:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1769074581.152150:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037ddf80 x1855007612386432/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074581 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074581.152160:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006136680 x1855007612386944/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074586 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074581.152186:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006134a80 x1855007612387456/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074592 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1769074581.152196:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1769074581.152199:0:2367:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1769074581.152201:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1769074581.152203:0:2367:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074581.152255:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612387456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074581.152322:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388224:192.168.204.102@tcp:9:kworker.0 00000100:00100000:0.0:1769074581.152426:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388352:192.168.204.102@tcp:9:kworker.0 00000100:00100000:2.0:1769074581.155328:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:0:1855007612387968:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074581.155377:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074581.155389:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612388096:192.168.204.102@tcp:400:kworker.0 00000100:00000400:3.0:1769074581.162691:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074565/real 1769074565] req@ffff8b10037df480 x1855007612386560/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074581 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074581.162779:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037df480 x1855007612386560/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074581 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074581.162791:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1769074581.162795:0:2367:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1769074581.162802:0:2367:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1769074581.162824:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612386432:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074581.162834:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612386560:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074581.163322:0:16087:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074581.163333:0:16087:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769074581.163337:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074581.163347:0:16087:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1512 00000100:00080000:2.0:1769074581.163358:0:16087:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769074581.163403:0:16087:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612388480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074581.163417:0:16087:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074581.163420:0:16087:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769074581.163422:0:16087:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074581.163426:0:16087:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1523 00000100:00080000:2.0:1769074581.163431:0:16087:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074581.163440:0:16087:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769074581.163450:0:16087:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59c00 x1855007612388608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074581.163764:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612388480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769074581.163878:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388608:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074581.163936:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1005d58000 x1855007612388224/t0(0) o9->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074591 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074581.163945:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1769074581.163949:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388224:192.168.204.102@tcp:9:kworker.0 00000100:00080000:3.0:1769074581.163962:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1005d5b800 x1855007612388352/t0(0) o9->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074591 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074581.163970:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1769074581.163984:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388352:192.168.204.102@tcp:9:kworker.0 00000100:00100000:1.0:1769074581.169769:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612386944:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074582.055296:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074583.073262:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074584.095908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074585.122277:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074586.143947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074586.207835:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074570/real 1769074570] req@ffff8b1006136a00 x1855007612387072/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074586 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074586.207881:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006136a00 x1855007612387072/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074586 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074586.207903:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 50, was 49): CONNECTING 00000100:00100000:0.0:1769074586.207929:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387072:192.168.204.102@tcp:400:kworker.0 00000100:00080000:3.0:1769074586.271936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074586.271952:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074586.271955:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074586.271959:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1769074587.167886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074588.191931:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074589.218273:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074590.240104:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074591.263931:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074591.391998:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074581/real 1769074581] req@ffff8b1005d5ad80 x1855007612388480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074591 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074591.392035:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612388480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074591 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074591.392046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074581/real 1769074581] req@ffff8b1005d59c00 x1855007612388608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074591 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074591.392058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59c00 x1855007612388608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074591 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074591.392077:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074591.392086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769074591.392090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612388480:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769074591.392290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074591.392296:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769074591.392300:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769074591.392305:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1574 00000100:00080000:1.0:1769074591.392315:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1769074591.392322:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769074591.392357:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612388736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769074591.392373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074591.392376:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074591.392440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074591.392457:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769074591.392462:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769074591.392498:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612388736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769074591.392847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074591.392851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769074591.392853:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769074591.392858:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1574 00000100:00080000:1.0:1769074591.392863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1769074591.392868:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769074591.392877:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769074591.392894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612388864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074591.392991:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388864:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769074591.403148:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:2.0:1769074591.403165:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1769074591.403174:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3062c6c to 0x143201ebf3063119 00000100:00080000:2.0:1769074591.416667:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1769074591.416686:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:3.0:1769074591.420709:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612388736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769074591.420911:0:33150:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:2.0:1769074591.420916:0:33150:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:2.0:1769074591.420921:0:33150:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:2.0:1769074591.420938:0:33150:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074591.420953:0:33150:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074591.420959:0:33150:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074591.420980:0:33150:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074591.420991:0:33150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d80431 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3062c7a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074591.421004:0:33150:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074591.421010:0:33150:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074591.421017:0:33150:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074591.421021:0:33150:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074591.421024:0:33150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d8043f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3062c88 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074591.421037:0:33150:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074591.421043:0:33150:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074591.421049:0:33150:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074591.421052:0:33150:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074591.421055:0:33150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80446 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3062dfb expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074591.421067:0:33150:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074591.421073:0:33150:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074591.421077:0:33150:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1769074591.421080:0:33150:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074591.421083:0:33150:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169a00/0x68a7113811d80438 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3062c81 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:2.0:1769074591.421097:0:33150:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:2.0:1769074591.421102:0:33150:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:2.0:1769074591.421104:0:33150:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1769074591.421107:0:33150:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:2.0:1769074591.421111:0:33150:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:2.0:1769074592.288303:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074592.352214:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074576/real 1769074576] req@ffff8b1006137100 x1855007612387584/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074592 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074592.352262:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006137100 x1855007612387584/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074592 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074592.352270:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 51, was 49): CONNECTING 00000100:00100000:0.0:1769074592.352288:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612387584:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074593.312931:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1769074593.376183:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1769074593.376191:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 23 10000000:01000000:3.0:1769074593.376197:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1769074593.376255:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.376267:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.376278:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.376306:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612388992:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.376530:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.388246:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612388992:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074593.388281:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.388294:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.388389:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389120:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.388478:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.446464:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389120:192.168.204.102@tcp:256:ll_cfg_requeue.0 10000000:01000000:3.0:1769074593.446526:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074593.446536:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.446547:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074593.446558:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1769074593.446561:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:3.0:1769074593.446564:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1769074593.446600:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.446609:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.446619:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.446670:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389248:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.446832:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.456686:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389248:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074593.456714:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.456727:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.456817:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389376:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.456948:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.461748:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389376:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.461842:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389504:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.462158:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.472139:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389504:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1769074593.473805:0:33151:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:3.0:1769074593.474084:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:3.0:1769074593.474107:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1769074593.474118:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.474130:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074593.474140:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1769074593.474142:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:3.0:1769074593.474145:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1769074593.474183:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.474192:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.474202:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.474303:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389632:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.474615:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.483918:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389632:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074593.483956:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.483969:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074593.484041:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389760:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074593.484119:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.497029:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389760:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074593.497742:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389888:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074593.497880:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074593.501608:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612389888:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1769074593.503321:0:33152:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1769074593.504036:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1769074593.504059:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1769074593.504069:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074593.504080:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074593.504091:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1769074593.504094:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:0.0:1769074593.504097:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1769074593.504140:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074593.504148:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074593.504158:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074593.504185:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390016:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074593.504302:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074593.509124:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390016:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074593.509152:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074593.509165:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074593.509244:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390144:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074593.509323:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074593.512572:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390144:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1769074593.512613:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1769074593.512621:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074593.512629:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074594.336478:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074595.360822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074595.564469:0:2357:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074595.564562:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624690560 00000100:00100000:0.0:1769074595.564571:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624690560 00000100:00100000:0.0:1769074595.564615:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 197 00000100:00100000:0.0:1769074595.564622:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006134380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:86724556-597d-4d0e-ac75-6d9d64145050+3:36284:x1855007624690560:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074595.564635:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074595.564746:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074595.564753:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006134380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:86724556-597d-4d0e-ac75-6d9d64145050+3:36284:x1855007624690560:12345-192.168.204.102@tcp:104: Request processed in 131us (296us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074595.564761:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 197 00010000:00010000:2.0:1769074595.565185:0:29101:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074595.565193:0:29101:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074595.565201:0:29101:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1006169000 already unused, calling callback (ffffffffc121fc20) 10000000:00010000:2.0:1769074595.565205:0:29101:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074595.565213:0:29101:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074595.565220:0:29101:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074595.565226:0:29101:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074595.565261:0:29101:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074595.565265:0:29101:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074595.565282:0:29101:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074595.565286:0:29101:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169000/0x68a7113811d80493 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063120 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074595.565302:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074595.565304:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074595.565305:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074595.565313:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074595.565328:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612390272:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074595.571993:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612390272:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074596.384188:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074596.512011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074596.512020:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769074596.512062:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074596.512084:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612390400:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074596.530568:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612390400:192.168.204.102@tcp:400:kworker.0 10000000:01000000:2.0:1769074597.344247:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074597.344259:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 23 10000000:01000000:2.0:1769074597.344276:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074597.344338:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074597.344352:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074597.344364:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074597.344402:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390528:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074597.344534:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074597.347585:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390528:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074597.347623:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074597.347683:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074597.347803:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390656:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074597.347918:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074597.359270:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612390656:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1769074597.359498:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1769074597.361550:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074597.361574:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074597.361590:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074597.408777:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074597.410162:0:33199:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 51, was 0): CONNECTING 00010000:00080000:1.0:1769074597.410200:0:33199:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8b1008679800@lustre-MDT0000-mdc-ffff8b101127b800: found existing conn 192.168.204.102@tcp, moved to head 00000100:00000400:2.0:1769074597.540168:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074581/real 1769074581] req@ffff8b1005d58380 x1855007612388096/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074597 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074597.540214:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58380 x1855007612388096/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074597 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074597.540226:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 51, was 49): CONNECTING 00000100:00100000:2.0:1769074597.540248:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612388096:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074598.432299:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074599.456040:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074600.483848:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074601.503951:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074601.631985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074601.631995:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074601.632022:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074601.632052:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612390784:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074601.638777:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612390784:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074602.528121:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074603.552103:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074604.578999:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074605.598368:0:33253:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074605.598382:0:33253:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074605.598405:0:33253:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074605.598472:0:33253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33253 initiator: MDT0 00010000:00010000:2.0:1769074605.598482:0:33253:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33253 initiator: MDT0 00010000:00010000:2.0:1769074605.598492:0:33253:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33253 initiator: MDT0 00000100:00080000:2.0:1769074605.598519:0:33253:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d58380 x1855007612390912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074605.598530:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 0 seconds 00010000:00010000:2.0:1769074605.600004:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074606.623703:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 0 seconds 00010000:00010000:2.0:1769074606.623926:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074606.752101:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074606.752109:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074606.752247:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612391040:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074606.752398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074591/real 1769074591] req@ffff8b1006134a80 x1855007612388864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074606 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074606.752425:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612388864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074606 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.752439:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074606.752460:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769074606.752464:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612388864:192.168.204.102@tcp:38:kworker.0 00000100:00080000:0.0:1769074606.753079:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074606.753085:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074606.753087:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074606.753094:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 0 00000100:00080000:0.0:1769074606.753105:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074606.753117:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074606.753138:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025c00 x1855007612391168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074606.753209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769074606.757542:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612391040:192.168.204.102@tcp:400:kworker.0 00010000:00080000:3.0:1769074606.762877:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1006025c00 x1855007612391168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074616 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074606.762894:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 73014444035 gen 1 00000100:00100000:3.0:1769074606.762920:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d59f80 x1855007612386304/t73014444038(73014444038) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074578 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.762932:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 18 00000100:00080000:3.0:1769074606.762949:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1769074606.762955:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074606.762958:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 73014444035 gen 1 00000100:00100000:3.0:1769074606.762962:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d59f80 x1855007612386304/t73014444038(73014444038) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074578 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.762969:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 73014444035 last 0 00000100:00080000:3.0:1769074606.762976:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d59f80 x1855007612386304/t73014444038(73014444038) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074578 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074606.762989:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769074606.763012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612386304:192.168.204.102@tcp:36:rm.0 00000100:00080000:3.0:1769074606.860337:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d59f80 x1855007612386304/t73014444038(73014444038) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074622 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.860352:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074606.860356:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 73014444035 gen 1 00000100:00100000:3.0:1769074606.860361:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d59f80 x1855007612386304/t73014444038(73014444038) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/456 e 0 to 0 dl 1769074622 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.860369:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 73014444035 last 73014444038 00000100:00080000:3.0:1769074606.860374:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:0.0:1769074606.862518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612386304:192.168.204.102@tcp:36:rm.0 00010000:00080000:1.0:1769074606.863384:0:33268:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:1.0:1769074606.863389:0:33268:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (2) 00000080:00200000:1.0:1769074606.863436:0:33268:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x109:0x0]: losing layout lock 00010000:00010000:1.0:1769074606.863474:0:33268:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d8045b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3063024 expref: -99 pid: 32333 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074606.863504:0:33268:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80454 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3063008 expref: -99 pid: 32330 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074606.863514:0:33268:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:1.0:1769074606.865868:0:33268:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:0.0:1769074606.866149:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391296:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:3.0:1769074606.995144:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 73014444038 gen 1 00000100:00080000:3.0:1769074606.995172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:3.0:1769074606.995185:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1006135c00 x1855007612391296/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074622 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.995194:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1006135c00 x1855007612391296/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074622 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074606.995201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:0.0:1769074606.995589:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:0.0:1769074606.995597:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000aa7484c7): req@ffff8b1005d58380 x1855007612390912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769074606.995619:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391296:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:2.0:1769074606.997169:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074607.002301:0:33253:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33253:1855007612390912:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074607.002329:0:33253:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002342:0:33253:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002353:0:33253:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002364:0:33253:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002375:0:33253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002382:0:33253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002389:0:33253:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074607.002400:0:33253:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d58380 x1855007612390912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074622 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074607.002414:0:33253:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002427:0:33253:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074607.002434:0:33253:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074607.002464:0:33253:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074607.002471:0:33253:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:2.0:1769074607.002477:0:33253:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074607.002480:0:33253:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fcb8 00010000:00010000:2.0:1769074607.002483:0:33253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.002491:0:33253:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074607.003943:0:33253:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074607.003951:0:33253:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074607.003969:0:33253:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074607.003983:0:33253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.003994:0:33253:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074607.004002:0:33253:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074607.004008:0:33253:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074607.004012:0:33253:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:2.0:1769074607.004017:0:33253:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fcb8 00010000:00010000:2.0:1769074607.004021:0:33253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.004029:0:33253:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074607.004153:0:33253:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074607.004164:0:33253:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074607.004170:0:33253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.004177:0:33253:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074607.004183:0:33253:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074607.004187:0:33253:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074607.004190:0:33253:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:2.0:1769074607.004193:0:33253:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fbb0 00010000:00010000:2.0:1769074607.004196:0:33253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074607.004202:0:33253:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074607.004210:0:33253:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074607.004217:0:33253:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074607.004303:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.004318:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00800000:00000004:2.0:1769074607.004324:0:33253:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1552, max_age 1599 00000100:00100000:2.0:1769074607.004364:0:33253:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33253:1855007612391424:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074607.005034:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074607.039241:0:33253:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33253:1855007612391424:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074607.039270:0:33253:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:2.0:1769074607.039366:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040245:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040259:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040264:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040268:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040271:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040275:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040278:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040282:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040285:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040289:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040292:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040296:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040299:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040303:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040306:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040310:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040313:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040316:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040332:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040336:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040341:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040345:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040348:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040352:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040355:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040359:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040362:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040366:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040749:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040758:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040761:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040765:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040768:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040772:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040775:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040779:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040783:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040786:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040790:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040793:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040796:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040800:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040803:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040807:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040810:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040815:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040818:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040822:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040826:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040830:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040833:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040837:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040841:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040845:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040848:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040852:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040856:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040859:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040862:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040866:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040870:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040874:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040878:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040882:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040886:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040890:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040892:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040896:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040899:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040903:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040907:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040910:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040914:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040917:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040920:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040924:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040927:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040931:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040935:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040938:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040942:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040946:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040949:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040953:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040956:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040960:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040963:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040967:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040970:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040973:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040977:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040980:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040983:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040987:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040990:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.040994:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.040997:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041001:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041004:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041008:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041011:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041015:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041019:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041023:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041026:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041030:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041033:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041036:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041040:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041044:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041047:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041051:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041054:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041058:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041062:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041066:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041069:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041073:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041076:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041079:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041082:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041086:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041088:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041092:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041095:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041097:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00800000:00000080:2.0:1769074607.041101:0:33253:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.041104:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.041110:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00020000:00000004:2.0:1769074607.041117:0:33253:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1552, max_age 1599 00000100:00080000:2.0:1769074607.041135:0:33253:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 918 00000100:00080000:2.0:1769074607.041148:0:33253:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074607.041159:0:33253:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 207880192, lost: 0. 00000100:00100000:2.0:1769074607.041185:0:33253:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59f80 x1855007612391552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074607.041233:0:33253:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d58380 x1855007612391680/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074607.041243:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 0 seconds 00000100:00100000:0.0:1769074607.041563:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391552:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074607.063511:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934747 gen 6 00000100:00080000:3.0:1769074607.063530:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b101127b800: connect to target with instance 2 00000100:00080000:3.0:1769074607.063550:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1769074607.063562:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074607.063576:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074607.063582:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1769074607.063586:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1769074607.063594:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074607.063596:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074607.063608:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000aa7484c7): req@ffff8b1005d58380 x1855007612391680/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074607.063634:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391552:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074607.064040:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074607.080616:0:33253:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33253:1855007612391680:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074607.080691:0:33253:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074607.080755:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.080781:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00020000:00000004:2.0:1769074607.081366:0:33253:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1552, max_age 1599 00000100:00080000:2.0:1769074607.081383:0:33253:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 918 00000100:00080000:2.0:1769074607.081396:0:33253:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074607.081405:0:33253:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 207880192, lost: 0. 00000100:00100000:2.0:1769074607.081425:0:33253:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612391808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074607.081467:0:33253:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d58380 x1855007612391936/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074607.081477:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 0 seconds 00000100:00100000:3.0:1769074607.081919:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391808:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074607.118743:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967450 gen 6 00000100:00080000:3.0:1769074607.118763:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b101127b800: connect to target with instance 1 00000100:00080000:3.0:1769074607.118782:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1769074607.118793:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074607.118799:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074607.118811:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1769074607.118814:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1769074607.118822:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074607.118824:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074607.118837:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000aa7484c7): req@ffff8b1005d58380 x1855007612391936/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074607.118868:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612391808:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074607.120511:0:33253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074607.136149:0:33253:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33253:1855007612391936:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074607.136179:0:33253:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074607.136256:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136275:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136283:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136287:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136291:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136295:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136299:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136303:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136307:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136310:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136520:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136528:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136533:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136536:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136550:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136554:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136557:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136561:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136564:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136567:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136571:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136574:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136577:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136581:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136585:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136588:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136592:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136595:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136599:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136602:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136606:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136609:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136612:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136615:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136619:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136622:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136626:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136629:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136633:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136658:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136664:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136901:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136908:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136911:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136914:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136917:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136920:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136923:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136927:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136929:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136933:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136935:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136939:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136942:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136946:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136948:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136951:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136954:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136957:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136960:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136963:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136967:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136971:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136975:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136979:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136983:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136986:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136990:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.136995:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.136997:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137001:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137004:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137008:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137011:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137015:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137018:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137022:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137026:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137029:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137032:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137036:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137039:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137042:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137045:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137050:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137053:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137056:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137059:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137062:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137066:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137069:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137072:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137076:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137079:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137082:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137085:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137088:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137090:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137093:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137188:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137194:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137198:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137201:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137204:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137207:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137210:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137213:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137217:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137220:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137223:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137227:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137231:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137234:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137237:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137242:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137245:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137249:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137252:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137257:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137260:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137265:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137268:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137272:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137276:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200080:2.0:1769074607.137280:0:33253:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe26ba6d00 00020000:00000080:2.0:1769074607.137283:0:33253:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8c00 uarg=0000000000000000 00000080:00200008:2.0:1769074607.137297:0:33253:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:2.0:1769074607.648398:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074608.672274:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074609.696224:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074610.721273:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074611.532423:0:33504:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:2.0:1769074611.743948:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074611.875575:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612392064:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074611.875707:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392320:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074611.875740:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074611.875747:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392192:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074611.875770:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074611.885253:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392192:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074611.885273:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074611.885285:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612392064:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074611.885300:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612392320:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074612.775080:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074613.796304:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074613.917710:0:33659:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:2.0:1769074614.816270:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074615.842042:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1769074615.848156:0:33697:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074615.848174:0:33697:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074615.848199:0:33697:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074615.848217:0:33697:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.848232:0:33697:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.848240:0:33697:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074615.848247:0:33697:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074615.848251:0:33697:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074615.848477:0:33697:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fcb8 00010000:00010000:1.0:1769074615.848483:0:33697:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.848492:0:33697:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074615.849899:0:33697:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074615.849907:0:33697:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074615.849925:0:33697:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074615.849940:0:33697:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.849961:0:33697:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.849968:0:33697:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074615.849975:0:33697:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074615.849978:0:33697:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074615.849985:0:33697:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fcb8 00010000:00010000:1.0:1769074615.849988:0:33697:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.849996:0:33697:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.850119:0:33697:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074615.850126:0:33697:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074615.850133:0:33697:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.850142:0:33697:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.850147:0:33697:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074615.850151:0:33697:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074615.850154:0:33697:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074615.850157:0:33697:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140d7fbb0 00010000:00010000:1.0:1769074615.850161:0:33697:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.850172:0:33697:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.850181:0:33697:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074615.850187:0:33697:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:1.0:1769074615.850293:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.850306:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00800000:00000004:1.0:1769074615.850312:0:33697:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1600, max_age 1608 00000100:00100000:1.0:1769074615.850350:0:33697:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392576:192.168.204.102@tcp:41:lfs.0 00000100:00100000:1.0:1769074615.850444:0:33697:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2db40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074615.855468:0:33697:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392576:192.168.204.102@tcp:41:lfs.0 00800000:00000004:1.0:1769074615.855498:0:33697:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:1.0:1769074615.855575:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855597:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855606:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855611:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855615:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855619:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855623:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855627:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855631:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855635:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855678:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855683:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855689:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855693:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855697:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855701:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855705:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855709:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855713:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855717:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855721:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855724:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855728:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855732:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855735:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855739:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855742:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855746:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855750:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855753:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855757:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855760:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855763:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855766:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855770:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855773:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855776:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855780:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855783:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855787:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855791:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855794:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855798:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855802:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855805:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855809:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855812:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855816:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855820:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855823:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855827:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855830:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855833:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855836:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855839:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855843:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855846:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855849:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855852:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855855:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855859:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855862:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855866:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855870:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855873:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855876:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855879:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855882:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855886:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855889:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855892:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855895:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855899:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855902:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855906:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855909:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855913:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855916:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855920:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855923:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855926:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855930:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855934:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855937:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855941:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855944:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855948:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855952:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855956:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855959:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855963:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855967:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855971:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855975:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855978:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855982:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855986:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855988:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.855993:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.855996:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856000:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856003:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856007:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856010:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856014:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856018:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856022:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856025:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856029:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856033:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856037:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856040:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856045:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856049:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856053:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856058:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856062:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856066:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856070:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856073:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856077:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856081:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856085:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856088:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856093:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856096:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856100:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00800000:00000080:1.0:1769074615.856104:0:33697:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074615.856108:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:1.0:1769074615.856114:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00020000:00000004:1.0:1769074615.856122:0:33697:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1600, max_age 1608 00000100:00100000:1.0:1769074615.856158:0:33697:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392704:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074615.856231:0:33697:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2db40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074615.860069:0:33697:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392704:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074615.860099:0:33697:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074615.860179:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:1.0:1769074615.860196:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00020000:00000004:1.0:1769074615.860205:0:33697:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1600, max_age 1608 00000100:00100000:1.0:1769074615.860245:0:33697:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392832:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074615.860333:0:33697:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2db40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074615.863288:0:33697:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:33697:1855007612392832:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074615.863316:0:33697:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074615.863384:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863412:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863420:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863425:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863430:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863433:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863437:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863440:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863443:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863446:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863450:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863455:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863458:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863462:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863466:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863469:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863473:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863477:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863481:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863484:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863489:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863492:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863496:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863500:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863504:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863508:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863511:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863514:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863518:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863522:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863526:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863529:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863533:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863537:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863540:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863544:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863548:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863551:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863555:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863558:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863561:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863565:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863568:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863571:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863575:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863590:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863595:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863599:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863603:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863606:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863610:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863613:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863617:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863620:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863624:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863627:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863631:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863634:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863664:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863669:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863674:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863677:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863681:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863684:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863687:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863690:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863694:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863697:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863701:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863704:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863707:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863711:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863714:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863716:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863719:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863722:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863725:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863728:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863732:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863735:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863739:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863742:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863746:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863749:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863753:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863756:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863760:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863763:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863766:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863769:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863772:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863775:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863778:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863781:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863785:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863788:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863792:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863795:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863799:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863802:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863806:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863809:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863813:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863817:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863821:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863825:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863829:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863832:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863836:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863839:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863843:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863846:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863850:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863853:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863857:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863860:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863864:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863867:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863871:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863874:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863878:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863881:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863885:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863889:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074615.863892:0:33697:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd80532e50 00020000:00000080:2.0:1769074615.863895:0:33697:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200008:2.0:1769074615.863909:0:33697:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:1.0:1769074615.911377:0:33714:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074615.911416:0:33714:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074615.911434:0:33714:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.911451:0:33714:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.911459:0:33714:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074615.911467:0:33714:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074615.911471:0:33714:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074615.911479:0:33714:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa91414fbb58 00010000:00010000:1.0:1769074615.911482:0:33714:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.911490:0:33714:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074615.911506:0:33714:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074615.911519:0:33714:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:1.0:1769074615.911523:0:33714:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:1.0:1769074615.911536:0:33714:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d6b.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074615.911584:0:33714:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33714 initiator: MDT0 00010000:00010000:1.0:1769074615.911592:0:33714:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33714 initiator: MDT0 00010000:00010000:1.0:1769074615.911600:0:33714:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33714 initiator: MDT0 00000100:00100000:1.0:1769074615.911628:0:33714:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:33714:1855007612392960:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:1.0:1769074615.911734:0:33714:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a2d540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074615.930197:0:33714:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:33714:1855007612392960:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:0.0:1769074615.930220:0:33714:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930233:0:33714:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930240:0:33714:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930247:0:33714:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930254:0:33714:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930260:0:33714:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8a00 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:0.0:1769074615.930264:0:33714:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930270:0:33714:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930276:0:33714:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930283:0:33714:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074615.930289:0:33714:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 33714 initiator: MDT0 00010000:00010000:0.0:1769074615.930295:0:33714:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8a00/0x68a7113811d804bd lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 33714 initiator: MDT0 00000002:00100000:0.0:1769074615.930311:0:33714:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1006136300 x1855007612392960/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074631 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'checkstat.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074615.930325:0:33714:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074615.930443:0:33714:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.930456:0:33714:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074615.930465:0:33714:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa91414fb940 00010000:00010000:1.0:1769074615.930469:0:33714:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074615.930476:0:33714:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074616.873683:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074616.995436:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612393088:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074616.995534:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393344:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074616.995560:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074616.995565:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393216:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074616.995584:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393472:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074617.001904:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393216:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074617.001924:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393472:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074617.001934:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612393088:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074617.001944:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612393344:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074617.887939:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074618.913712:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074619.935780:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074620.966937:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074621.983974:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074622.120430:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612393600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074622.120519:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074622.120557:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074622.120564:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393728:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074622.120587:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074622.131070:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612393600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074622.154105:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393728:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074622.154125:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074622.154138:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612393856:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074623.010486:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1769074623.368703:0:34279:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 7: mkdir |X| contained create ====== 04:37:01 (1769074621) 00000080:00200000:0.0:1769074623.623187:0:34291:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074623.623210:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074623.623224:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.623236:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074623.623240:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074623.623247:0:34291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074623.623249:0:34291:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:0.0:1769074623.623311:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87b58 00010000:00010000:0.0:1769074623.623316:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.623323:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074623.623339:0:34291:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074623.623357:0:34291:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=13 00000080:00200000:0.0:1769074623.623361:0:34291:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:0.0:1769074623.623374:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074623.623421:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 34291 initiator: MDT0 00010000:00010000:0.0:1769074623.623427:0:34291:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34291 initiator: MDT0 00010000:00010000:0.0:1769074623.623437:0:34291:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34291 initiator: MDT0 00000100:00100000:0.0:1769074623.623462:0:34291:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34291:1855007612394112:192.168.204.102@tcp:101:lfs.0 00000100:00100000:0.0:1769074623.623557:0:34291:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074623.648086:0:34291:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34291:1855007612394112:192.168.204.102@tcp:101:lfs.0 00010000:00010000:1.0:1769074623.648110:0:34291:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648124:0:34291:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648131:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648138:0:34291:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648144:0:34291:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648149:0:34291:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ac600 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:1.0:1769074623.648154:0:34291:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648160:0:34291:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648167:0:34291:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648174:0:34291:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074623.648180:0:34291:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 34291 initiator: MDT0 00010000:00010000:1.0:1769074623.648186:0:34291:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac600/0x68a7113811d804c4 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 34291 initiator: MDT0 00000002:00100000:1.0:1769074623.648201:0:34291:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1006026d80 x1855007612394112/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074639 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074623.648212:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074623.648233:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648242:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074623.648247:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87940 00010000:00010000:1.0:1769074623.648250:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648255:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.648361:0:34291:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074623.648374:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074623.648380:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648387:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.648393:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074623.648397:0:34291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074623.648401:0:34291:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074623.648405:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87be8 00010000:00010000:1.0:1769074623.648407:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648414:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.648427:0:34291:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074623.648434:0:34291:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=16392 00000080:00200008:1.0:1769074623.648459:0:34291:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074623.648628:0:34291:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074623.648879:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074623.648888:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648897:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.648903:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074623.648908:0:34291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074623.648910:0:34291:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074623.648914:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87cb8 00010000:00010000:1.0:1769074623.648917:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.648924:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074623.649506:0:34291:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074623.649512:0:34291:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074623.649522:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074623.649531:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.649540:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.649545:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074623.649549:0:34291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074623.649551:0:34291:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:1.0:1769074623.649554:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87cb8 00010000:00010000:1.0:1769074623.649557:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074623.649561:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074623.649697:0:34291:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074623.649705:0:34291:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074623.649713:0:34291:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.649766:0:34291:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074623.649774:0:34291:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074623.649780:0:34291:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074623.649782:0:34291:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804b6 00000080:00010000:0.0:1769074623.649787:0:34291:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804b6 from it ffffaa9140c87bb0 00010000:00010000:0.0:1769074623.649790:0:34291:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.649796:0:34291:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074623.649806:0:34291:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:0.0:1769074623.649813:0:34291:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00000080:00200080:0.0:1769074623.650121:0:34291:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c03066f0 arg=7fff602c9270 00000080:00200000:0.0:1769074623.650137:0:34291:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) name=d7.replay-single stripe_offset=0 stripe_count=1, hash_type=0 00010000:00010000:0.0:1769074623.650163:0:34291:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.650226:0:34291:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074623.650232:0:34291:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074623.650235:0:34291:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d804b6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30632d2 expref: -99 pid: 33253 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074623.650257:0:34291:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074623.650285:0:34291:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006024e00 x1855007612394368/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074623.650309:0:34291:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34291:1855007612394368:192.168.204.102@tcp:36:lfs.0 00000100:00100000:0.0:1769074623.650398:0:34291:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074623.673174:0:34291:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34291:1855007612394368:192.168.204.102@tcp:36:lfs.0 00000080:00200000:2.0:1769074623.673223:0:34291:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00200000:2.0:1769074623.673230:0:34291:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 512 00000080:00200008:2.0:1769074623.673272:0:34291:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 300000 00010000:00010000:0.0:1769074624.032840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074624.896985:0:34300:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074624.897000:0:34300:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074624.897022:0:34300:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074624.897084:0:34300:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 34300 initiator: MDT0 00010000:00010000:0.0:1769074624.897093:0:34300:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34300 initiator: MDT0 00010000:00010000:0.0:1769074624.897102:0:34300:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34300 initiator: MDT0 00000100:00100000:0.0:1769074624.897129:0:34300:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394496:192.168.204.102@tcp:101:lfs.0 00000100:00100000:0.0:1769074624.897222:0:34300:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074624.915886:0:34300:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394496:192.168.204.102@tcp:101:lfs.0 00010000:00010000:0.0:1769074624.915914:0:34300:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915926:0:34300:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915938:0:34300:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915950:0:34300:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915961:0:34300:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915970:0:34300:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.915978:0:34300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074624.915988:0:34300:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006027480 x1855007612394496/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074624.916001:0:34300:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.916013:0:34300:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074624.916018:0:34300:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074624.916045:0:34300:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:0.0:1769074624.916051:0:34300:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:0.0:1769074624.916056:0:34300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074624.916074:0:34300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d23cb8 00010000:00010000:0.0:1769074624.916078:0:34300:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.916087:0:34300:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074624.917748:0:34300:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074624.917758:0:34300:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074624.917777:0:34300:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074624.917792:0:34300:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.917806:0:34300:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074624.917815:0:34300:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074624.917822:0:34300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074624.917825:0:34300:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:0.0:1769074624.917830:0:34300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d23cb8 00010000:00010000:0.0:1769074624.917835:0:34300:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.917842:0:34300:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074624.918586:0:34300:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074624.918601:0:34300:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074624.918610:0:34300:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.918620:0:34300:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074624.918626:0:34300:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074624.918631:0:34300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074624.918634:0:34300:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:0.0:1769074624.918884:0:34300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d23bb0 00010000:00010000:0.0:1769074624.918889:0:34300:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074624.918896:0:34300:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074624.918923:0:34300:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:0.0:1769074624.918932:0:34300:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:0.0:1769074624.919218:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.919233:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00800000:00000004:0.0:1769074624.919253:0:34300:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1609, max_age 1617 00000100:00100000:0.0:1769074624.919295:0:34300:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394624:192.168.204.102@tcp:41:lfs.0 00000100:00100000:0.0:1769074624.919374:0:34300:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074624.941012:0:34300:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394624:192.168.204.102@tcp:41:lfs.0 00800000:00000004:0.0:1769074624.941042:0:34300:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:0.0:1769074624.941129:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941146:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941154:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941158:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941162:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941166:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941176:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941181:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941184:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941188:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941192:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941195:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941198:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941201:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941206:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941209:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941213:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941216:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941221:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941224:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941230:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941233:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941236:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941249:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941255:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941259:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941263:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941266:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941270:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941273:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941277:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941280:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941285:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941289:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941293:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941296:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941300:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941303:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941307:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941311:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941315:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941319:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941323:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941326:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941330:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941334:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941338:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941342:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941346:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941349:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941353:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941356:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941360:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941364:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941368:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941371:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941375:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941378:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941382:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941385:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941389:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941392:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941397:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941401:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941405:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941409:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941412:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941416:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941420:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941423:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941427:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941431:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941434:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941438:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941442:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941445:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941449:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941452:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941456:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941458:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941462:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941465:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941468:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941472:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941475:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941479:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941483:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941486:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941489:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941493:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941497:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941500:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941503:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941507:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941510:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941514:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941518:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941521:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941525:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941528:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941531:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941534:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941537:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941540:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941543:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941546:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941549:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941552:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941555:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941558:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941561:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941564:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941567:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941570:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941574:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941577:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941580:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941582:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941586:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941589:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941592:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941596:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941599:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941602:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941606:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941608:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941612:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00800000:00000080:0.0:1769074624.941615:0:34300:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.941619:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.941625:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00020000:00000004:0.0:1769074624.941632:0:34300:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1609, max_age 1617 00000100:00100000:0.0:1769074624.941713:0:34300:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394752:192.168.204.102@tcp:13:lfs.0 00000100:00100000:0.0:1769074624.941778:0:34300:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074624.953395:0:34300:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394752:192.168.204.102@tcp:13:lfs.0 00020000:00000004:0.0:1769074624.953423:0:34300:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:0.0:1769074624.953496:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.953514:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00020000:00000004:0.0:1769074624.953522:0:34300:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1609, max_age 1617 00000100:00100000:0.0:1769074624.953561:0:34300:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394880:192.168.204.102@tcp:13:lfs.0 00000100:00100000:0.0:1769074624.953691:0:34300:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074624.967928:0:34300:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34300:1855007612394880:192.168.204.102@tcp:13:lfs.0 00020000:00000004:0.0:1769074624.967959:0:34300:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:0.0:1769074624.968036:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968054:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968063:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968067:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968071:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968075:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968079:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968083:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968086:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968090:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968094:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968097:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968101:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968105:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968109:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968113:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968116:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968119:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968123:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968126:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968129:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968133:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968136:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968139:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968142:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968146:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968149:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968151:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968155:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968158:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968161:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968165:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968168:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968172:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968175:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968179:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968183:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968186:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968189:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968193:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968196:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968199:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968202:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968205:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968208:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968211:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968215:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968218:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968221:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968224:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968227:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968230:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968233:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968237:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968253:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968257:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968261:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968265:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968269:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968273:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968276:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968280:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968284:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968287:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968291:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968294:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968298:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968301:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968305:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968308:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968312:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968317:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968320:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968324:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968328:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968331:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968335:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968339:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968342:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968346:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968349:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968352:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968356:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968359:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968363:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968366:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968370:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968373:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968377:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968380:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968383:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968386:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968390:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968393:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968396:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968399:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968403:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968406:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968409:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968412:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968415:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968419:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968423:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968426:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968430:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968435:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968440:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968444:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968448:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968451:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968454:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968458:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968462:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968473:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968477:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968480:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968484:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968487:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968491:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968494:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968498:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968501:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968505:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968509:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200080:0.0:1769074624.968512:0:34300:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff902aeff0 00020000:00000080:0.0:1769074624.968516:0:34300:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006028400 uarg=0000000000000000 00000080:00200008:0.0:1769074624.968535:0:34300:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:1.0:1769074625.056313:0:34307:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074625.056342:0:34307:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074625.056360:0:34307:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056377:0:34307:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056386:0:34307:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074625.056394:0:34307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074625.056398:0:34307:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:1.0:1769074625.056405:0:34307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d53bd0 00010000:00010000:1.0:1769074625.056410:0:34307:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056419:0:34307:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056434:0:34307:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074625.056450:0:34307:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:1.0:1769074625.056454:0:34307:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074625.056459:0:34307:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074625.056465:0:34307:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056473:0:34307:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056480:0:34307:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074625.056484:0:34307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074625.056487:0:34307:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:1.0:1769074625.056491:0:34307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d53cd0 00010000:00010000:1.0:1769074625.056495:0:34307:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056502:0:34307:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056511:0:34307:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074625.056516:0:34307:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074625.056523:0:34307:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074625.056529:0:34307:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056537:0:34307:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056542:0:34307:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074625.056547:0:34307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074625.056549:0:34307:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804cb 00000080:00010000:1.0:1769074625.056553:0:34307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804cb from it ffffaa9141d53db0 00010000:00010000:1.0:1769074625.056556:0:34307:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.056563:0:34307:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074625.056571:0:34307:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074625.056576:0:34307:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:1.0:1769074625.056580:0:34307:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:1.0:1769074625.056606:0:34307:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.057255:0:34307:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074625.057263:0:34307:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074625.057266:0:34307:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad200/0x68a7113811d804cb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30634f4 expref: -99 pid: 34300 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074625.057294:0:34307:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074625.057307:0:34307:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135f80 x1855007612395136/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074625.057332:0:34307:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34307:1855007612395136:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:1.0:1769074625.057417:0:34307:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5d7480 going to sleep for 11 seconds 00010000:00010000:2.0:1769074625.057922:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074625.063169:0:34307:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34307:1855007612395136:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074625.063214:0:34307:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffff8b100486cb08) 00000080:00200000:2.0:1769074625.063262:0:34307:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:0.0:1769074625.076073:0:34310:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074625.076101:0:34310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074625.076161:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 34310 initiator: MDT0 00010000:00010000:0.0:1769074625.076173:0:34310:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34310 initiator: MDT0 00010000:00010000:0.0:1769074625.076185:0:34310:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34310 initiator: MDT0 00000100:00100000:0.0:1769074625.076209:0:34310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395264:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074625.076311:0:34310:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074625.080347:0:34310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395264:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074625.080372:0:34310:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080384:0:34310:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080394:0:34310:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080408:0:34310:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080418:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080425:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080432:0:34310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074625.080444:0:34310:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006027480 x1855007612395264/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074641 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074625.080532:0:34310:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080545:0:34310:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074625.080552:0:34310:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074625.080582:0:34310:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074625.080588:0:34310:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804d2 00000080:00010000:2.0:1769074625.080593:0:34310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074625.080596:0:34310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804d2 from it ffffaa9141d53b58 00010000:00010000:2.0:1769074625.080599:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.080607:0:34310:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.080631:0:34310:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074625.080687:0:34310:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074625.080692:0:34310:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074625.080712:0:34310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074625.080751:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 34310 initiator: MDT0 00010000:00010000:2.0:1769074625.080758:0:34310:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34310 initiator: MDT0 00010000:00010000:2.0:1769074625.080767:0:34310:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34310 initiator: MDT0 00000100:00100000:2.0:1769074625.080787:0:34310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395392:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074625.080848:0:34310:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a06c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074625.093398:0:34310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395392:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074625.093426:0:34310:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093449:0:34310:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093457:0:34310:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093468:0:34310:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093483:0:34310:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093495:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093501:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093507:0:34310:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074625.093517:0:34310:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5ad80 x1855007612395392/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074641 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074625.093529:0:34310:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093538:0:34310:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/269 (0 0) 00000080:00200000:2.0:1769074625.093544:0:34310:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074625.093567:0:34310:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10d:0x0], updating i_size 0 00000080:00200000:2.0:1769074625.093570:0:34310:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10d:0x0](ffff8b100486cb08): rc = 0 00000080:00200000:2.0:1769074625.093586:0:34310:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x10d:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1769074625.093592:0:34310:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10d:0x0](ffff8b100486cb08) for lock 0x68a7113811d804d9 00000080:00010000:2.0:1769074625.093603:0:34310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10d:0x0](ffff8b100486cb08) 00000080:00010000:2.0:1769074625.093606:0:34310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804d9 from it ffffaa9141d53b20 00010000:00010000:2.0:1769074625.093610:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093629:0:34310:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074625.093708:0:34310:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x10d:0x0](ffff8b100486cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074625.093714:0:34310:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffff8b100486cb08),name=fsa-oleg402-client.virtnet 00000002:00010000:2.0:1769074625.093734:0:34310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074625.093741:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093752:0:34310:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093758:0:34310:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074625.093762:0:34310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10d:0x0](ffff8b100486cb08) 00000080:00010000:2.0:1769074625.093765:0:34310:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10d:0x0](ffff8b100486cb08) for lock 0x68a7113811d804d9 00000080:00010000:2.0:1769074625.093769:0:34310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804d9 from it ffffaa9141d53cb8 00010000:00010000:2.0:1769074625.093771:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093778:0:34310:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093834:0:34310:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074625.093842:0:34310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074625.093860:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093872:0:34310:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093877:0:34310:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074625.093882:0:34310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074625.093885:0:34310:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804d2 00000080:00010000:2.0:1769074625.093889:0:34310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804d2 from it ffffaa9141d53c10 00010000:00010000:2.0:1769074625.093892:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093899:0:34310:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093906:0:34310:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074625.093915:0:34310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:2.0:1769074625.093919:0:34310:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074625.093925:0:34310:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074625.093930:0:34310:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093938:0:34310:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093943:0:34310:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074625.093946:0:34310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074625.093948:0:34310:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804d2 00000080:00010000:2.0:1769074625.093952:0:34310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804d2 from it ffffaa9141d53d68 00010000:00010000:2.0:1769074625.093954:0:34310:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.093961:0:34310:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.093968:0:34310:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074625.093973:0:34310:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074625.093991:0:34310:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.094011:0:34310:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074625.094021:0:34310:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x10d:0x0]: losing layout lock 00010000:00010000:2.0:1769074625.094049:0:34310:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.094145:0:34310:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.094152:0:34310:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074625.094155:0:34310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d804d9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306352c expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074625.094171:0:34310:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029acc00/0x68a7113811d804d2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063510 expref: -99 pid: 34310 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074625.094190:0:34310:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074625.094198:0:34310:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5ad80 x1855007612395648/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074625.094225:0:34310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395648:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074625.094418:0:34310:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a06c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074625.101372:0:34310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:34310:1855007612395648:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074625.101433:0:34310:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x10d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074625.101444:0:34310:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x10d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074625.101452:0:34310:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffff8b100486cb08) 00010000:00010000:2.0:1769074626.079990:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074627.104109:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074627.231895:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612395904:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.232030:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396160:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.232069:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074627.232076:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612395776:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.232110:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396032:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.237243:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612395776:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.237279:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396032:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.237302:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612395904:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074627.237323:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396160:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074628.128475:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074628.224887:0:34341:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1769074628.253550:0:34342:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074628.253581:0:34342:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074628.254323:0:34342:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 34342 initiator: MDT0 00010000:00010000:2.0:1769074628.254339:0:34342:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34342 initiator: MDT0 00010000:00010000:2.0:1769074628.254351:0:34342:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34342 initiator: MDT0 00000100:00100000:2.0:1769074628.254383:0:34342:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396288:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:2.0:1769074628.255668:0:34342:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a06c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074628.272099:0:34342:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396288:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:2.0:1769074628.272126:0:34342:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272139:0:34342:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272156:0:34342:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272167:0:34342:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272179:0:34342:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272199:0:34342:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272207:0:34342:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074628.272218:0:34342:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5b800 x1855007612396288/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074644 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074628.272233:0:34342:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272246:0:34342:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074628.272253:0:34342:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074628.272281:0:34342:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074628.272287:0:34342:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804e0 00000080:00010000:2.0:1769074628.272293:0:34342:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074628.272296:0:34342:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804e0 from it ffffaa914164bbd0 00010000:00010000:2.0:1769074628.272300:0:34342:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.272309:0:34342:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074628.272343:0:34342:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074628.272363:0:34342:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=24 00000080:00200000:2.0:1769074628.272368:0:34342:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=0 00000002:00010000:2.0:1769074628.272386:0:34342:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1769074628.272426:0:34342:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 34342 initiator: MDT0 00010000:00010000:2.0:1769074628.272434:0:34342:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34342 initiator: MDT0 00010000:00010000:2.0:1769074628.272442:0:34342:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34342 initiator: MDT0 00000100:00100000:2.0:1769074628.272461:0:34342:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396416:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:2.0:1769074628.272536:0:34342:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a06c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074628.278929:0:34342:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396416:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:2.0:1769074628.278959:0:34342:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.278989:0:34342:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.278998:0:34342:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279016:0:34342:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279028:0:34342:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279041:0:34342:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279050:0:34342:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279057:0:34342:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074628.279071:0:34342:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff8b1005d5b800 x1855007612396416/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074644 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074628.279087:0:34342:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279097:0:34342:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:2.0:1769074628.279105:0:34342:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074628.279131:0:34342:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 11776 00000080:00200000:2.0:1769074628.279134:0:34342:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffff8b100486db88): rc = 0 00000080:00010000:2.0:1769074628.279141:0:34342:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d804e7 00000080:00010000:2.0:1769074628.279154:0:34342:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:2.0:1769074628.279158:0:34342:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804e7 from it ffffaa914164baa0 00010000:00010000:2.0:1769074628.279163:0:34342:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074628.279171:0:34342:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074628.279219:0:34342:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074628.279244:0:34342:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88), flags=1536 00000080:00200000:2.0:1769074628.279249:0:34342:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 3 00000080:00200000:2.0:1769074628.279257:0:34342:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 3 00000080:00200000:2.0:1769074628.279261:0:34342:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88), flags=33188, excl=1 00000080:00200000:2.0:1769074628.279265:0:34342:0:(namei.c:2020:ll_mknod()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88) mode 100644 dev 0 00000020:00100000:2.0:1769074628.279343:0:34342:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074628.279351:0:34342:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5b800 x1855007612396672/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074628.279374:0:34342:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396672:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074628.279446:0:34342:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a06c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074628.297676:0:34342:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:34342:1855007612396672:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074628.297735:0:34342:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffff8b100486cb08) 00000080:00200000:2.0:1769074628.297783:0:34342:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f7.replay-single, unhashed 1 00010000:00010000:2.0:1769074629.153215:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074630.178978:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074631.201365:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074632.234565:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074632.352269:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612396800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074632.352395:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612397056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074632.353237:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074632.353308:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612397184:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074632.364389:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612397056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074632.364471:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612397184:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074633.248361:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074634.271922:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074635.296153:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074636.323470:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074637.344755:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074637.472522:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397440:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074637.472674:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397696:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074637.472718:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074637.472725:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612397312:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074637.472759:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397568:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074637.483888:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397568:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074637.483914:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397696:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074638.372090:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074639.392038:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074640.416221:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074641.441401:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074642.464002:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074642.592622:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397952:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074642.592836:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612398208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074642.592874:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074642.592881:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612397824:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074642.593057:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612398080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074642.609280:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612398080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074642.609301:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612398208:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074643.488155:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074644.511803:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074645.536056:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074646.566867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074647.583972:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074647.712168:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:1.0:1769074647.712180:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b101127b800: disconnect after 23s idle 00000100:00100000:3.0:1769074647.712249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398592:192.168.204.102@tcp:9:kworker.0 00000100:00080000:1.0:1769074647.712490:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:1.0:1769074647.712494:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b101127b800: disconnect after 23s idle 00000100:00100000:0.0:1769074647.713442:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612398464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074647.713551:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074647.713562:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612398336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074647.723341:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398720:192.168.204.102@tcp:9:kworker.0 00000100:00080000:3.0:1769074647.731717:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006137b80 x1855007612398592/t0(0) o9->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074657 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074647.731747:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1769074647.731752:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398592:192.168.204.102@tcp:9:kworker.0 00000100:00080000:3.0:1769074647.751214:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006135c00 x1855007612398720/t0(0) o9->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074657 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074647.751233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1769074647.751238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398720:192.168.204.102@tcp:9:kworker.0 00010000:00010000:0.0:1769074648.608399:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074648.735966:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074632/real 1769074632] req@ffff8b1006136300 x1855007612396800/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074648 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074648.736011:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006136300 x1855007612396800/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074648 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074648.736021:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1769074648.736027:0:2365:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00000400:1.0:1769074648.736222:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074632/real 1769074632] req@ffff8b1006135880 x1855007612396928/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074648 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074648.736242:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006135880 x1855007612396928/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074648 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074648.736250:0:2364:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074648.736254:0:2364:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074648.736261:0:2364:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074648.736302:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612396928:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074648.760923:0:2365:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:0.0:1769074648.760933:0:2365:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:0.0:1769074648.760946:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006136300 x1855007612396800/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074648 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074648.760966:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d58000 x1855007612397312/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074653 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074648.761063:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037ded80 x1855007612397824/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074658 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074648.761083:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006137800 x1855007612398336/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074663 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1769074648.761110:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1769074648.761114:0:2365:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:0.0:1769074648.761116:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:0.0:1769074648.761119:0:2365:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074648.769460:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612398336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074648.769549:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612397312:192.168.204.102@tcp:400:kworker.0 00000100:00080000:3.0:1769074648.769613:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074648.769622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769074648.769628:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074648.769636:0:38:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1584 00000100:00080000:3.0:1769074648.769666:0:38:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769074648.769789:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612398848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074648.769820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074648.769824:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074648.769826:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074648.769830:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1599 00000100:00080000:3.0:1769074648.769836:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769074648.769865:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612398848:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769074648.770006:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074648.770021:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dfb80 x1855007612398976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074648.770045:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769074648.772132:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612396800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074648.791183:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612397824:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074649.640145:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074650.662369:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074651.680025:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074652.709690:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074652.768720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074652.768735:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074652.768738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074652.768742:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1769074653.729096:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074653.791784:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074653.791799:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074653.791802:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074653.791805:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:2.0:1769074653.921480:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074637/real 1769074637] req@ffff8b1005d5bb80 x1855007612397440/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074653 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074653.921527:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d5bb80 x1855007612397440/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074653 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074653.921538:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 53, was 52): CONNECTING 00000100:00100000:2.0:1769074653.921561:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397440:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074654.752040:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074655.776056:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074656.799891:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074657.824107:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074658.015811:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074642/real 1769074642] req@ffff8b10037ddf80 x1855007612397952/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074658.015859:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037ddf80 x1855007612397952/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074658.015870:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 53, was 52): CONNECTING 00000100:00100000:2.0:1769074658.015889:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612397952:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074658.849139:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074658.977180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074648/real 1769074648] req@ffff8b10037dfb80 x1855007612398976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074658.977225:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dfb80 x1855007612398976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074658.977237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074648/real 1769074648] req@ffff8b10037dc000 x1855007612398848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074658.977249:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612398848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074658 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074658.977273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769074658.977300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769074658.977304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612398976:192.168.204.102@tcp:38:kworker.0 00000100:00080000:0.0:1769074658.977310:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769074658.977317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769074658.977320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612398848:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074658.978702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074658.978715:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769074658.978720:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074658.978732:0:38:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1641 00000100:00080000:3.0:1769074658.978749:0:38:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769074658.978758:0:38:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769074658.978808:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612399104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074658.978854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074658.978858:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074658.978860:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074658.978864:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1641 00000100:00080000:3.0:1769074658.978869:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769074658.978873:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074658.978885:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074658.978907:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612399232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074658.979003:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612399104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769074658.979129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612399232:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074658.985167:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:3.0:1769074658.985193:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1769074658.985207:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3063119 to 0x143201ebf30635a3 00000100:00080000:3.0:1769074658.985213:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1769074658.985220:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:0.0:1769074658.986684:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612399104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074658.986870:0:34531:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:3.0:1769074658.986875:0:34531:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1769074658.986880:0:34531:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1769074658.986914:0:34531:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074658.986928:0:34531:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074658.986936:0:34531:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074658.987013:0:34531:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074658.987026:0:34531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d8049a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063127 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074658.987044:0:34531:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074658.987052:0:34531:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074658.987058:0:34531:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074658.995184:0:34531:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074658.995221:0:34531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029ada00/0x68a7113811d804a8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063135 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074658.995262:0:34531:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074658.995270:0:34531:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074658.995276:0:34531:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074659.003667:0:34531:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074659.003702:0:34531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d804af lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf30632a8 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074659.003741:0:34531:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074659.003750:0:34531:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074659.003756:0:34531:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1769074659.003763:0:34531:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074659.003767:0:34531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804a1 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf306312e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:3.0:1769074659.003788:0:34531:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1769074659.003796:0:34531:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1769074659.003800:0:34531:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1769074659.003804:0:34531:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1769074659.003813:0:34531:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:3.0:1769074659.876394:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074660.896699:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:2.0:1769074661.023982:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074661.023994:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 24 10000000:01000000:2.0:1769074661.024002:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074661.024083:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.024097:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.024111:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074661.024146:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399360:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.024292:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074661.037276:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399360:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074661.037316:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.037332:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074661.037445:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399488:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.037692:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074661.081124:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399488:192.168.204.102@tcp:256:ll_cfg_requeue.0 10000000:01000000:2.0:1769074661.081204:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1769074661.081218:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.081232:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074661.081244:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1769074661.081247:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:2.0:1769074661.081251:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1769074661.081308:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.081317:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.081327:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074661.081354:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399616:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.081462:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074661.091239:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399616:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074661.091293:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.091311:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074661.091431:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399744:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.092613:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074661.101240:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399744:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.101443:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399872:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074661.101575:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074661.115589:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612399872:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1769074661.121154:0:34532:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:3.0:1769074661.121869:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:3.0:1769074661.121898:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1769074661.121913:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.121928:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074661.121939:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1769074661.121942:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:3.0:1769074661.121945:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1769074661.122000:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.122009:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.122021:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074661.122058:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400000:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.122201:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074661.127036:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400000:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074661.127080:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.127102:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074661.127219:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400128:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.127373:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074661.141877:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400128:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.141990:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400256:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.142131:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074661.147989:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400256:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1769074661.162617:0:34533:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:3.0:1769074661.163694:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:3.0:1769074661.163728:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1769074661.163744:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.163760:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074661.163775:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:3.0:1769074661.163779:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:3.0:1769074661.163782:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:3.0:1769074661.163841:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.163850:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.163861:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074661.163909:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400384:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.164026:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074661.185762:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400384:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074661.185809:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.185828:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074661.185944:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400512:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074661.186039:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074661.199061:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400512:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1769074661.199129:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1769074661.199143:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074661.199156:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074661.259934:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074661.260060:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624706816 00000100:00100000:1.0:1769074661.260081:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624706816 00000100:00100000:1.0:1769074661.260141:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 198 00000100:00100000:1.0:1769074661.260153:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:37734:x1855007624706816:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074661.260168:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074661.260262:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074661.260269:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:37734:x1855007624706816:12345-192.168.204.102@tcp:104: Request processed in 119us (345us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074661.260277:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 198 00010000:00010000:2.0:1769074661.260331:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074661.260346:0:13998:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.260358:0:13998:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698000 already unused, calling callback (ffffffffc121fc20) 10000000:00010000:2.0:1769074661.260363:0:13998:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.260373:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074661.260381:0:13998:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074661.260388:0:13998:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074661.260456:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.260465:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074661.260495:0:13998:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.260502:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d804ee lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063675 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074661.260524:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074661.260526:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074661.260528:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074661.260551:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612400640:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074661.269541:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612400640:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074661.920164:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1769074662.879779:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1769074662.879787:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 24 10000000:01000000:3.0:1769074662.879793:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1769074662.879841:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074662.879853:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074662.879864:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074662.879892:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400768:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074662.880012:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074662.891664:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400768:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074662.891695:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074662.891707:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074662.891802:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400896:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1769074662.891875:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037eccc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074662.903864:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612400896:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1769074662.903930:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1769074662.904063:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074662.904074:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074662.904084:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074662.944138:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769074662.970505:0:34580:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 54, was 0): CONNECTING 00010000:00080000:0.0:1769074662.970540:0:34580:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8b1008679800@lustre-MDT0000-mdc-ffff8b101127b800: found existing conn 192.168.204.102@tcp, moved to head 00000100:00000400:0.0:1769074663.073123:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074647/real 1769074647] req@ffff8b1006135180 x1855007612398464/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074663 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074663.073158:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006135180 x1855007612398464/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074663 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074663.073168:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 54, was 52): CONNECTING 00000100:00100000:0.0:1769074663.073185:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612398464:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074663.970000:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074664.095955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074664.095962:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074664.096392:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612401024:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074664.103345:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612401024:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074664.995172:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074666.017279:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074667.059102:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074668.064109:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074669.087957:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074669.216438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074669.216445:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074669.216513:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612401152:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074669.219150:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612401152:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074670.112012:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074670.978938:0:34632:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074670.978954:0:34632:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074670.978976:0:34632:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074670.978993:0:34632:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074670.979007:0:34632:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074670.979018:0:34632:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074670.979025:0:34632:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074670.979029:0:34632:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804e0 00000080:00010000:2.0:1769074670.979036:0:34632:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804e0 from it ffffaa9140dd3cb8 00010000:00010000:2.0:1769074670.979041:0:34632:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074670.979049:0:34632:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074670.987064:0:34632:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074670.987075:0:34632:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074670.987108:0:34632:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074670.987128:0:34632:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074670.987144:0:34632:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074670.987152:0:34632:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074670.987160:0:34632:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074670.987164:0:34632:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804e0 00000080:00010000:3.0:1769074670.987171:0:34632:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804e0 from it ffffaa9140dd3cb8 00010000:00010000:3.0:1769074670.987175:0:34632:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074670.987184:0:34632:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074670.987324:0:34632:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074670.987332:0:34632:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074670.987339:0:34632:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074670.987348:0:34632:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074670.987355:0:34632:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074670.987360:0:34632:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074670.987363:0:34632:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d804e0 00000080:00010000:3.0:1769074670.987367:0:34632:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d804e0 from it ffffaa9140dd3bb0 00010000:00010000:3.0:1769074670.987370:0:34632:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074670.987377:0:34632:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074670.987385:0:34632:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074670.987394:0:34632:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074670.987732:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074670.987748:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00800000:00000004:3.0:1769074670.987755:0:34632:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1618, max_age 1663 00000100:00080000:3.0:1769074670.987799:0:34632:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b10037dd500 x1855007612401280/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074670.987810:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 0 seconds 00010000:00010000:3.0:1769074671.136009:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074672.032677:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 0 seconds 00010000:00010000:0.0:1769074672.159950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074673.055886:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 0 seconds 00010000:00010000:0.0:1769074673.183889:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074673.314167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074658/real 1769074658] req@ffff8b10037dca80 x1855007612399232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074673 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074673.314203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612399232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074673 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.314223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769074673.314254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769074673.314259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612399232:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769074673.315629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074673.315636:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769074673.315675:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074673.315684:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 0 00000100:00080000:2.0:1769074673.315700:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074673.315712:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769074673.315728:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612401536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074673.315766:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074673.315781:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612401408:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074673.316704:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769074673.325725:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612401408:192.168.204.102@tcp:400:kworker.0 00010000:00080000:1.0:1769074673.333255:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1005d58000 x1855007612401536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074683 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074673.333276:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 77309411331 gen 1 00000100:00100000:1.0:1769074673.333307:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5b800 x1855007612396672/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074644 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.333325:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 19 00000100:00080000:1.0:1769074673.333578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1769074673.333586:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074673.333589:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 77309411331 gen 1 00000100:00100000:1.0:1769074673.333595:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5b800 x1855007612396672/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074644 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.333602:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 77309411331 last 0 00000100:00080000:1.0:1769074673.333611:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d5b800 x1855007612396672/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074644 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074673.333625:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074673.334214:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612396672:192.168.204.102@tcp:36:mcreate.0 00000100:00080000:1.0:1769074673.394617:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d5b800 x1855007612396672/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074689 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.394691:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074673.394696:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 77309411331 gen 1 00000100:00100000:1.0:1769074673.394702:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5b800 x1855007612396672/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074689 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.394711:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 77309411331 last 77309411332 00000100:00080000:1.0:1769074673.394717:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:0.0:1769074673.395483:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612396672:192.168.204.102@tcp:36:mcreate.0 00010000:00080000:1.0:1769074673.395593:0:34648:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:1.0:1769074673.395599:0:34648:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (2) 00010000:00010000:1.0:1769074673.395684:0:34648:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699a00/0x68a7113811d804e0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3063556 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074673.395732:0:34648:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698800/0x68a7113811d804e7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3063572 expref: -99 pid: 34342 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074673.395743:0:34648:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:1.0:1769074673.396197:0:34648:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:0.0:1769074673.396600:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401664:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:1.0:1769074673.517461:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 77309411332 gen 1 00000100:00080000:1.0:1769074673.517501:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:1.0:1769074673.517520:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1006135f80 x1855007612401664/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074689 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.517532:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1006135f80 x1855007612401664/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074689 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074673.517542:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:0.0:1769074673.524519:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:0.0:1769074673.524544:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000105cf854): req@ffff8b10037dd500 x1855007612401280/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074673.524579:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401664:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:3.0:1769074673.524954:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074673.549063:0:34632:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34632:1855007612401280:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074673.549107:0:34632:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00200080:3.0:1769074673.549217:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549239:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549248:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549254:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549259:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549264:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549268:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549272:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549276:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549280:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549284:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549289:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549293:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549297:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549301:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549304:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549309:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549312:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549317:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549320:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549325:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549328:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549332:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549336:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549340:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549344:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549361:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549366:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549370:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549374:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549379:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549383:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549387:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549391:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549396:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.549400:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.549404:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550582:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550858:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550866:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550871:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550874:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550878:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550881:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550884:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550886:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550889:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550892:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550895:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550897:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550900:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550903:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550905:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550908:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550911:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550915:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550918:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550921:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550924:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550926:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550929:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550932:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550935:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550937:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550940:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550943:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550946:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550948:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550951:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550953:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550956:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550959:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550962:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550966:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550969:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550972:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550976:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550979:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550982:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550986:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550990:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.550994:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.550997:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551001:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551005:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551008:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551014:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551017:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551022:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551025:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551029:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551033:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551037:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551040:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551044:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551057:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551062:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551064:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551068:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551071:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551074:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551077:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551079:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551083:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551086:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551089:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551092:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551095:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551098:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551101:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551104:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551106:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551109:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551112:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551115:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551117:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551120:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551123:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551126:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551128:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551131:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551134:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551136:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551139:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551142:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551144:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551147:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00800000:00000080:3.0:1769074673.551171:0:34632:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.551176:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.551183:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00020000:00000004:3.0:1769074673.551194:0:34632:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1618, max_age 1665 00000100:00080000:3.0:1769074673.551217:0:34632:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1600 00000100:00080000:3.0:1769074673.551236:0:34632:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074673.551251:0:34632:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:3.0:1769074673.551286:0:34632:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612401792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074673.551343:0:34632:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b10037dd500 x1855007612401920/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074673.551353:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769074673.553437:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401792:192.168.204.102@tcp:8:lfs.0 00000100:00100000:1.0:1769074673.570276:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934747 gen 7 00000100:00080000:1.0:1769074673.570302:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b101127b800: connect to target with instance 2 00000100:00080000:1.0:1769074673.570315:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:1.0:1769074673.570331:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1769074673.570338:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:1.0:1769074673.570345:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:1.0:1769074673.570350:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:1.0:1769074673.570358:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:1.0:1769074673.570360:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:1.0:1769074673.570377:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000105cf854): req@ffff8b10037dd500 x1855007612401920/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074673.570412:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612401792:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074673.575841:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074673.609312:0:34632:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34632:1855007612401920:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074673.609357:0:34632:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074673.609595:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.609627:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00020000:00000004:3.0:1769074673.609696:0:34632:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1618, max_age 1665 00000100:00080000:3.0:1769074673.609714:0:34632:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1600 00000100:00080000:3.0:1769074673.609727:0:34632:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074673.609740:0:34632:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:3.0:1769074673.609771:0:34632:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612402048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074673.609811:0:34632:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b10037dd500 x1855007612402176/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074673.609820:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 0 seconds 00000100:00100000:0.0:1769074673.610089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612402048:192.168.204.102@tcp:8:lfs.0 00000100:00100000:1.0:1769074673.614238:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967450 gen 7 00000100:00080000:1.0:1769074673.614264:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b101127b800: connect to target with instance 1 00000100:00080000:1.0:1769074673.614277:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:0.0:1769074673.614385:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:0.0:1769074673.614395:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:0.0:1769074673.614477:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:0.0:1769074673.614484:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:0.0:1769074673.614494:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:0.0:1769074673.614497:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:0.0:1769074673.614514:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000105cf854): req@ffff8b10037dd500 x1855007612402176/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074673.614828:0:34632:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c563c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074673.615557:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612402048:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074673.630259:0:34632:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:34632:1855007612402176:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074673.630301:0:34632:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074673.630398:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630441:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630451:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630456:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630461:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630464:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630468:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630472:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630476:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630479:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630483:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630486:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630490:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630493:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630497:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630501:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630504:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630507:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630511:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630514:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630518:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630522:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630525:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630530:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630534:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630537:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630541:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630544:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630548:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630552:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630556:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630559:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630563:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630566:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630570:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630573:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630577:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630580:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630584:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630587:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630591:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630594:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.630598:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.630601:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631384:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631393:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631397:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631401:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631405:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631408:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631412:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631415:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631420:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631436:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631441:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631445:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631449:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631453:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631457:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631460:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631464:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631468:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631471:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631475:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631479:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631483:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631487:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631490:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631494:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631498:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631501:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631505:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631509:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631514:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631518:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631522:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631527:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631530:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631535:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631538:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631542:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631546:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631549:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631552:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631556:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631559:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631563:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631566:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631570:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631574:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631578:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631583:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631587:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631591:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.631595:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.631598:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632527:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632548:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632563:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632568:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632572:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632576:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632580:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632584:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632588:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632591:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632595:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632625:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.632632:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.632635:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634179:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634202:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634209:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634213:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634218:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634221:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634226:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634229:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634233:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634237:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634242:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634246:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634250:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634254:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074673.634258:0:34632:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffce821d800 00020000:00000080:3.0:1769074673.634262:0:34632:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200008:3.0:1769074673.634281:0:34632:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:2.0:1769074674.208780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074675.234218:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074676.265331:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074677.287445:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074678.304414:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074678.439774:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402432:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.439861:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402688:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.439894:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074678.439911:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612402304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.439945:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.448275:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612402304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.448299:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.448311:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402688:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074678.451192:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612402432:192.168.204.102@tcp:400:kworker.0 00000001:02000400:2.0:1769074678.975814:0:34886:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:3.0:1769074679.332508:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074680.354076:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074681.380487:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074682.040239:0:35041:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:0.0:1769074682.399864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074683.423922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074683.552171:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612402944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.552284:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612403200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.552318:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1769074683.552325:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612402816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.552357:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612403072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.568486:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612402816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.568814:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612403072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.568826:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612402944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074683.568837:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612403200:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074684.452616:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1769074684.593085:0:35079:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074684.593101:0:35079:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074684.593121:0:35079:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.593180:0:35079:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35079 initiator: MDT0 00010000:00010000:3.0:1769074684.593192:0:35079:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35079 initiator: MDT0 00010000:00010000:3.0:1769074684.593201:0:35079:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35079 initiator: MDT0 00000100:00100000:3.0:1769074684.593227:0:35079:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403328:192.168.204.102@tcp:101:lfs.0 00000100:00100000:3.0:1769074684.593367:0:35079:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.603565:0:35079:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403328:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074684.603595:0:35079:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603607:0:35079:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603619:0:35079:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603630:0:35079:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603688:0:35079:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603697:0:35079:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603705:0:35079:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.603718:0:35079:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dca80 x1855007612403328/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074684.603735:0:35079:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603750:0:35079:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074684.603757:0:35079:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074684.603791:0:35079:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074684.603798:0:35079:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:3.0:1769074684.603804:0:35079:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074684.603807:0:35079:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa9140d9bcb8 00010000:00010000:3.0:1769074684.603810:0:35079:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.603818:0:35079:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074684.605310:0:35079:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074684.605319:0:35079:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074684.605337:0:35079:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074684.605352:0:35079:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074684.605365:0:35079:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074684.605371:0:35079:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074684.605378:0:35079:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074684.605382:0:35079:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:2.0:1769074684.605388:0:35079:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa9140d9bcb8 00010000:00010000:2.0:1769074684.605391:0:35079:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074684.605398:0:35079:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074684.607467:0:35079:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074684.607497:0:35079:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074684.607514:0:35079:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074684.607530:0:35079:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074684.607538:0:35079:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074684.607545:0:35079:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074684.607549:0:35079:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:2.0:1769074684.607556:0:35079:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa9140d9bbb0 00010000:00010000:2.0:1769074684.607560:0:35079:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074684.607567:0:35079:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074684.607583:0:35079:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074684.607593:0:35079:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074684.609151:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:2.0:1769074684.609174:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00800000:00000004:2.0:1769074684.609181:0:35079:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1666, max_age 1676 00000100:00100000:2.0:1769074684.609230:0:35079:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403456:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074684.609334:0:35079:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074684.616849:0:35079:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403456:192.168.204.102@tcp:41:lfs.0 00800000:00000004:0.0:1769074684.616883:0:35079:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00200080:0.0:1769074684.616963:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.616981:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.616988:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.616992:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.616997:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617000:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617004:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617008:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617013:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617016:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617020:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617023:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617026:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617029:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617033:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617035:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617039:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617043:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617046:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617050:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617053:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617058:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617062:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617065:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617070:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617074:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617077:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617081:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617085:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617089:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617092:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617095:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617099:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617102:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617106:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617109:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617112:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617115:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617118:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617122:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617126:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617129:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617135:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617139:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617145:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617148:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617152:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617155:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617158:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617161:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617165:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617169:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617172:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617176:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617180:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617183:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617187:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617190:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617194:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617197:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617201:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617204:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617208:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617211:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617217:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617220:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617226:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617229:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617232:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617235:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617251:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617256:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617261:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617264:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617268:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617271:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617275:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617279:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617283:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617285:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617289:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617292:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617296:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617299:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617302:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617306:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617309:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617312:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617315:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617319:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617323:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617326:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617330:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617333:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617337:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617340:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617344:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617347:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617351:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617354:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617358:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617362:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617365:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617368:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617372:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617376:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617379:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617382:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617386:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617390:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617393:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617397:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617401:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617404:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617409:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617412:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617416:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617420:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617423:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617426:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617430:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617433:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617437:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617440:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617443:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617446:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617450:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00800000:00000080:0.0:1769074684.617452:0:35079:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00000080:00200080:0.0:1769074684.617456:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:0.0:1769074684.617461:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602b800 uarg=0000000000000000 00020000:00000004:0.0:1769074684.617468:0:35079:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1666, max_age 1676 00000100:00100000:0.0:1769074684.617506:0:35079:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403584:192.168.204.102@tcp:13:lfs.0 00000100:00100000:0.0:1769074684.617591:0:35079:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364cb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074684.631630:0:35079:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403584:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074684.631696:0:35079:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074684.631774:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.631791:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00020000:00000004:2.0:1769074684.631798:0:35079:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1666, max_age 1676 00000100:00100000:2.0:1769074684.631838:0:35079:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403712:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074684.631941:0:35079:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2c00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074684.638604:0:35079:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35079:1855007612403712:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074684.638634:0:35079:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074684.638743:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638760:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638767:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638771:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638776:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638779:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638783:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638786:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638790:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638793:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638811:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638815:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638819:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638822:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638826:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638829:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638832:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638836:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638839:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638843:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638846:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638850:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638853:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638856:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638860:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638873:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638877:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638881:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638885:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638889:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638893:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638896:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638900:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638903:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638906:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638910:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638913:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638916:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638919:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638922:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638925:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638928:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638932:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638935:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638939:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638942:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638946:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638950:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638953:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638957:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638961:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638964:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638969:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638972:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638975:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638978:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638982:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638985:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638989:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.638993:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.638997:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639001:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639005:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639008:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639012:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639016:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639020:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639024:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639027:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639030:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639033:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639036:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639039:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639042:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639046:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639049:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639052:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639056:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639059:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639062:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639066:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639069:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639072:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639076:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639079:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639082:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639086:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639089:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639093:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639097:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639101:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639104:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639108:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639111:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639115:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639119:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639122:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639126:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639129:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639132:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639135:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639138:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639143:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639145:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639148:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639151:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639155:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639157:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639160:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639163:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639166:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639169:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639173:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639176:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639179:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639182:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639185:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639188:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639192:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639195:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639198:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639201:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639204:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639207:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074684.639211:0:35079:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffdd087bc70 00020000:00000080:2.0:1769074684.639214:0:35079:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200008:2.0:1769074684.639227:0:35079:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:0.0:1769074684.663118:0:35095:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074684.663146:0:35095:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.663162:0:35095:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.663180:0:35095:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.663189:0:35095:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074684.663197:0:35095:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074684.663201:0:35095:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:0.0:1769074684.663208:0:35095:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa9140c87b58 00010000:00010000:0.0:1769074684.663212:0:35095:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.663227:0:35095:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.663256:0:35095:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074684.663272:0:35095:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:0.0:1769074684.663277:0:35095:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:0.0:1769074684.663291:0:35095:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.663337:0:35095:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35095 initiator: MDT0 00010000:00010000:0.0:1769074684.663344:0:35095:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35095 initiator: MDT0 00010000:00010000:0.0:1769074684.663353:0:35095:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35095 initiator: MDT0 00000100:00100000:0.0:1769074684.663379:0:35095:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:35095:1855007612403840:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:0.0:1769074684.663490:0:35095:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.679307:0:35095:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:35095:1855007612403840:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:3.0:1769074684.679333:0:35095:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679355:0:35095:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679364:0:35095:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679376:0:35095:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679394:0:35095:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679406:0:35095:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679428:0:35095:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679436:0:35095:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.679448:0:35095:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006027b80 x1855007612403840/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074684.679461:0:35095:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679482:0:35095:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:3.0:1769074684.679489:0:35095:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074684.679809:0:35095:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 11776 00000080:00200000:3.0:1769074684.679814:0:35095:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffff8b100486db88): rc = 0 00000080:00010000:3.0:1769074684.679821:0:35095:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80518 00000080:00010000:3.0:1769074684.679832:0:35095:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:3.0:1769074684.679835:0:35095:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80518 from it ffffaa9140c87b20 00010000:00010000:3.0:1769074684.679839:0:35095:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679852:0:35095:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074684.679895:0:35095:0:(file.c:6252:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074684.679901:0:35095:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88),name=d7.replay-single 00000002:00010000:3.0:1769074684.679919:0:35095:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.679927:0:35095:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.679937:0:35095:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.679943:0:35095:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074684.679948:0:35095:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:3.0:1769074684.679951:0:35095:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80518 00000080:00010000:3.0:1769074684.679955:0:35095:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80518 from it ffffaa9140c87cb8 00010000:00010000:3.0:1769074684.680833:0:35095:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.680846:0:35095:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.714510:0:35097:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074684.714537:0:35097:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.714555:0:35097:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.714574:0:35097:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.714582:0:35097:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074684.714590:0:35097:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074684.714595:0:35097:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:0.0:1769074684.714602:0:35097:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa9141c8bb58 00010000:00010000:0.0:1769074684.714606:0:35097:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.714615:0:35097:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.714630:0:35097:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074684.714655:0:35097:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=24 00000080:00200000:0.0:1769074684.714662:0:35097:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074684.714673:0:35097:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88), flags=8 00000080:00200000:0.0:1769074684.714677:0:35097:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88), intent=getattr 00000002:00010000:0.0:1769074684.714689:0:35097:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f7.replay-single,[0x0:0x0:0x0]) in obj [0x200001b71:0x10c:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.714735:0:35097:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35097 initiator: MDT0 00010000:00010000:0.0:1769074684.714743:0:35097:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35097 initiator: MDT0 00010000:00010000:0.0:1769074684.714752:0:35097:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35097 initiator: MDT0 00000100:00100000:0.0:1769074684.714776:0:35097:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:35097:1855007612403968:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:0.0:1769074684.716536:0:35097:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074684.725978:0:35097:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:35097:1855007612403968:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:1.0:1769074684.726008:0:35097:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726036:0:35097:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726044:0:35097:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726054:0:35097:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726065:0:35097:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726078:0:35097:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726087:0:35097:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726096:0:35097:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074684.726106:0:35097:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006027100 x1855007612403968/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074684.726117:0:35097:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726127:0:35097:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/270 (0 0) 00000080:00200000:1.0:1769074684.726134:0:35097:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x10c:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074684.726158:0:35097:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10e:0x0], updating i_size 0 00000080:00200000:1.0:1769074684.726162:0:35097:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10e:0x0](ffff8b100486cb08): rc = 0 00000080:00200000:1.0:1769074684.726178:0:35097:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x10e:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1769074684.726184:0:35097:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10e:0x0](ffff8b100486cb08) for lock 0x68a7113811d8051f 00000080:00010000:1.0:1769074684.726194:0:35097:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10e:0x0](ffff8b100486cb08) 00000080:00010000:1.0:1769074684.726197:0:35097:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8051f from it ffffaa9141c8bb20 00010000:00010000:1.0:1769074684.726202:0:35097:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726211:0:35097:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074684.726264:0:35097:0:(file.c:6252:ll_getattr_dentry()) START file f7.replay-single:[0x200001b71:0x10e:0x0](ffff8b100486cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074684.726271:0:35097:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffff8b100486cb08),name=f7.replay-single 00000002:00010000:1.0:1769074684.726291:0:35097:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10e:0x0]) in obj [0x200001b71:0x10c:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074684.726299:0:35097:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726309:0:35097:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074684.726315:0:35097:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x10c:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074684.726319:0:35097:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10e:0x0](ffff8b100486cb08) 00000080:00010000:1.0:1769074684.726322:0:35097:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10e:0x0](ffff8b100486cb08) for lock 0x68a7113811d8051f 00000080:00010000:1.0:1769074684.726325:0:35097:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8051f from it ffffaa9141c8bcb8 00010000:00010000:1.0:1769074684.726328:0:35097:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.726334:0:35097:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.744914:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074684.744943:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.744962:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.744983:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.744991:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074684.744998:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074684.745002:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:0.0:1769074684.745008:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa914156fb58 00010000:00010000:0.0:1769074684.745012:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.745020:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.745035:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074684.745043:0:35099:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=8 00000080:00200008:0.0:1769074684.745051:0:35099:0:(file.c:6252:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074684.745056:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88),name=d7.replay-single 00000002:00010000:0.0:1769074684.745067:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.745074:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.745081:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.745087:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074684.745091:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:0.0:1769074684.745094:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80518 00000080:00010000:0.0:1769074684.745098:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80518 from it ffffaa914156fcb8 00010000:00010000:0.0:1769074684.745101:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.745108:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.745135:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074684.745141:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074684.745147:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.745156:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.745162:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074684.745166:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074684.745168:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:0.0:1769074684.745172:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa914156fbf8 00010000:00010000:0.0:1769074684.745175:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074684.745181:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074684.745188:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074684.745192:0:35099:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=266 00000080:00200000:0.0:1769074684.745196:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074684.745215:0:35099:0:(file.c:1001:ll_file_open()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), flags 704400 00000002:00010000:0.0:1769074684.745231:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:0.0:1769074684.745291:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35099 initiator: MDT0 00010000:00010000:0.0:1769074684.745298:0:35099:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00000020:00100000:0.0:1769074684.745309:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074684.745321:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006027800 x1855007612404224/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074684.745331:0:35099:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00000100:00100000:0.0:1769074684.745351:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404224:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074684.745599:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c840 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.770175:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404224:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074684.770200:0:35099:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770213:0:35099:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770220:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770228:0:35099:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770245:0:35099:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770252:0:35099:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10029ac000 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:3.0:1769074684.770257:0:35099:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770264:0:35099:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770271:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770278:0:35099:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074684.770285:0:35099:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.770290:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d80526 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35099 initiator: MDT0 00000002:00100000:3.0:1769074684.770308:0:35099:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006027800 x1855007612404224/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074684.770320:0:35099:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006027800 x1855007612404224/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1769074684.770332:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074684.770354:0:35099:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 11776 00000002:00100000:3.0:1769074684.770371:0:35099:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006027800 x1855007612404224/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1769074684.770466:0:35099:0:(file.c:6252:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074684.770474:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88),name=d7.replay-single 00000002:00010000:3.0:1769074684.770494:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.770519:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.770530:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.770538:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074684.770544:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:3.0:1769074684.770548:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80518 00000080:00010000:3.0:1769074684.770554:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80518 from it ffffaa914156fcd8 00010000:00010000:3.0:1769074684.770557:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.770565:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.770660:0:35099:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88) pos/size0/11776 32bit_api 0 00000002:00010000:3.0:1769074684.770673:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200001b71:0x10c:0x0], intent: readdir flags 0 00010000:00010000:3.0:1769074684.770681:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.770689:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074684.772377:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404352:192.168.204.102@tcp:37:rm.0 00000100:00100000:3.0:1769074684.772476:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:1.0:1769074684.803318:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404352:192.168.204.102@tcp:37:rm.0 00010000:00010000:1.0:1769074684.805077:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.805097:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074684.805211:0:35099:0:(file.c:424:ll_file_release()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), flags 704000 00000002:00100000:1.0:1769074684.805290:0:35099:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006027800 x1855007612404224/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074684.805302:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074684.805308:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135880 x1855007612404608/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074684.805329:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404608:192.168.204.102@tcp:35:rm.0 00000100:00100000:1.0:1769074684.805727:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009ac90c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074684.814558:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404608:192.168.204.102@tcp:35:rm.0 00000002:00100000:1.0:1769074684.814594:0:35099:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1006027800 x1855007612404224/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074684.814683:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074684.814701:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074684.814717:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.814730:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074684.814739:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074684.814744:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074684.814748:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:1.0:1769074684.814755:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa914156fbf8 00010000:00010000:1.0:1769074684.814759:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074684.814769:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074684.814783:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074684.814790:0:35099:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=266 00000080:00200000:1.0:1769074684.814797:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074684.814804:0:35099:0:(file.c:1001:ll_file_open()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), flags 704400 00000002:00010000:1.0:1769074684.814819:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240704401 00010000:00010000:1.0:1769074684.814865:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35099 initiator: MDT0 00010000:00010000:1.0:1769074684.814873:0:35099:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00000020:00100000:1.0:1769074684.814883:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074684.814889:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135880 x1855007612404864/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074684.814899:0:35099:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00000100:00100000:1.0:1769074684.814918:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404864:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074684.815125:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009ac90c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.828866:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612404864:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074684.828903:0:35099:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.828914:0:35099:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.828925:0:35099:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.828936:0:35099:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.828952:0:35099:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b1006135880 x1855007612404864/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074684.828965:0:35099:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006135880 x1855007612404864/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074684.828977:0:35099:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.828988:0:35099:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 16 res 8589941617/268 (0 0) 00000080:00200000:3.0:1769074684.828994:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074684.829016:0:35099:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 11776 00000080:00010000:3.0:1769074684.829020:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d8052d 00000080:00010000:3.0:1769074684.829029:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8052d from it ffffaa914156fbd0 00010000:00010000:3.0:1769074684.829032:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829041:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.829065:0:35099:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006135880 x1855007612404864/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1769074684.829140:0:35099:0:(file.c:6252:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074684.829148:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88),name=d7.replay-single 00000002:00010000:3.0:1769074684.829169:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.829191:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829203:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.829210:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074684.829216:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:3.0:1769074684.829219:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80518 00000080:00010000:3.0:1769074684.829223:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80518 from it ffffaa914156fcd8 00010000:00010000:3.0:1769074684.829226:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829233:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.829327:0:35099:0:(llite_lib.c:2766:ll_statfs()) VFS Op:sb=lustre (ffff8b101127b800) d7.replay-single proj=0 00000080:00000004:3.0:1769074684.829332:0:35099:0:(obd_class.h:1083:obd_statfs()) lustre-clilmv-ffff8b101127b800: age 641, max_age 1677 00800000:00000004:3.0:1769074684.829341:0:35099:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1677, max_age 1677 00800000:00000004:3.0:1769074684.829344:0:35099:0:(obd_class.h:1123:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: use ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00000004:3.0:1769074684.829348:0:35099:0:(obd_class.h:1109:obd_statfs()) lustre-clilmv-ffff8b101127b800: update ffff8b1007d01c48 cache blocks 17229/17271 objects 500913/501262 00000080:00000004:3.0:1769074684.829351:0:35099:0:(llite_lib.c:2629:ll_statfs_internal()) MDC blocks 17229/17271 objects 500913/501262 00000080:00000004:3.0:1769074684.829355:0:35099:0:(obd_class.h:1083:obd_statfs()) lustre-clilov-ffff8b101127b800: age -421, max_age 1677 00020000:00000004:3.0:1769074684.829367:0:35099:0:(obd_class.h:1049:obd_statfs_async()) lustre-OST0000-osc-ffff8b101127b800: age 1677, max_age 1677 00000008:00000004:3.0:1769074684.829372:0:35099:0:(osc_request.c:3371:osc_statfs_async()) lustre-OST0000-osc-ffff8b101127b800: use ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00020000:00000004:3.0:1769074684.829377:0:35099:0:(obd_class.h:1049:obd_statfs_async()) lustre-OST0001-osc-ffff8b101127b800: age 1677, max_age 1677 00000008:00000004:3.0:1769074684.829379:0:35099:0:(osc_request.c:3371:osc_statfs_async()) lustre-OST0001-osc-ffff8b101127b800: use ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00000004:3.0:1769074684.829389:0:35099:0:(obd_class.h:1109:obd_statfs()) lustre-clilov-ffff8b101127b800: update ffff8b1007d066d0 cache blocks 7356/7366 objects 235520/236380 00000080:00000004:3.0:1769074684.829392:0:35099:0:(llite_lib.c:2640:ll_statfs_internal()) OSC blocks 7356/7366 objects 235520/236380 00000080:00200000:3.0:1769074684.829406:0:35099:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88) pos/size0/11776 32bit_api 0 00000080:00200000:3.0:1769074684.829428:0:35099:0:(dir.c:354:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88) pos/size18446744073709551614/11776 32bit_api 0 00000080:00200000:3.0:1769074684.829447:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 1 00000080:00200000:3.0:1769074684.829456:0:35099:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f7.replay-single, flags=0 00000080:00200000:3.0:1769074684.829462:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88), inode mode 41ed mask 3 00000080:00200000:3.0:1769074684.829467:0:35099:0:(namei.c:2354:ll_unlink()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffff8b100486db88) 00010000:00010000:3.0:1769074684.829636:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829724:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829786:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.829799:0:35099:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x10e:0x0]: losing layout lock 00010000:00010000:3.0:1769074684.829831:0:35099:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829839:0:35099:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829846:0:35099:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074684.829849:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad600/0x68a7113811d8051f lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063969 expref: -99 pid: 35097 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.829864:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80518 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306394d expref: -99 pid: 35095 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074684.829880:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074684.829887:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de300 x1855007612405120/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074684.829907:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405120:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074684.831422:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.863902:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405120:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074684.863958:0:35099:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x10e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074684.863970:0:35099:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x10e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074684.863976:0:35099:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffff8b100486cb08) 00000080:00200008:3.0:1769074684.864034:0:35099:0:(file.c:424:ll_file_release()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffff8b100486db88), flags 704000 00010000:00010000:3.0:1769074684.864056:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.864101:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074684.864115:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.864124:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.864135:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.864142:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074684.864147:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074684.864150:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:3.0:1769074684.864156:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa914156fc10 00010000:00010000:3.0:1769074684.864160:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.864167:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.864183:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074684.864201:0:35099:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=0 00000080:00200000:3.0:1769074684.864206:0:35099:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074684.864219:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.864284:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.864292:0:35099:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00010000:00010000:3.0:1769074684.864305:0:35099:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35099 initiator: MDT0 00000100:00100000:3.0:1769074684.864323:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405248:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074684.864388:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.896291:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405248:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074684.896321:0:35099:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896339:0:35099:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896349:0:35099:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896361:0:35099:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896375:0:35099:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896387:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896396:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896405:0:35099:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.896429:0:35099:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dc380 x1855007612405248/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074684.896444:0:35099:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896454:0:35099:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:3.0:1769074684.896462:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074684.896492:0:35099:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 11776 00000080:00200000:3.0:1769074684.896496:0:35099:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffff8b100486db88): rc = 0 00000080:00010000:3.0:1769074684.896502:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) for lock 0x68a7113811d80534 00000080:00010000:3.0:1769074684.896513:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffff8b100486db88) 00000080:00010000:3.0:1769074684.896517:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80534 from it ffffaa914156fe00 00010000:00010000:3.0:1769074684.896521:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896531:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.896562:0:35099:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074684.896578:0:35099:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074684.896588:0:35099:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896599:0:35099:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.896606:0:35099:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074684.896612:0:35099:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074684.896615:0:35099:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80511 00000080:00010000:3.0:1769074684.896619:0:35099:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80511 from it ffffaa914156fd88 00010000:00010000:3.0:1769074684.896622:0:35099:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896629:0:35099:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074684.896825:0:35099:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074684.896836:0:35099:0:(namei.c:2264:ll_rmdir()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074684.896858:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896881:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.896895:0:35099:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074684.896936:0:35099:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006135880 x1855007612404864/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1769074684.896949:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1769074684.896955:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc380 x1855007612405504/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1769074684.896977:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405504:192.168.204.102@tcp:35:rm.0 00000100:00100000:3.0:1769074684.897044:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074684.908822:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405504:192.168.204.102@tcp:35:rm.0 00000002:00100000:3.0:1769074684.908856:0:35099:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1006135880 x1855007612404864/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/608 e 0 to 0 dl 1769074700 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1769074684.908951:0:35099:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.908960:0:35099:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 1/0,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.908967:0:35099:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.908973:0:35099:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1769074684.908976:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d80534 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30639c4 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.908987:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d8052d lrc: 0/0,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063985 expref: -99 pid: 35099 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074684.908999:0:35099:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169a00/0x68a7113811d80511 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063931 expref: -99 pid: 35079 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074684.909016:0:35099:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074684.909022:0:35099:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc380 x1855007612405760/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074684.909043:0:35099:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405760:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074684.909119:0:35099:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074684.945536:0:35099:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35099:1855007612405760:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074684.945582:0:35099:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffff8b100486db88) 00010000:00010000:3.0:1769074685.472076:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074686.496814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074687.521574:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074688.548438:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074688.672589:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612405888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.672732:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406144:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.672771:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074688.672777:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406016:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.672816:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406272:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.680982:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406272:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.681004:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612405888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.681016:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406144:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074688.685160:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406016:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074689.576925:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074690.591820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074691.617155:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074692.644850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074693.663807:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074693.795741:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612406400:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.795831:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406656:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.795877:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074693.795884:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406528:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.795909:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406784:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.806494:0:2366:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378630 gen 1 00000100:00100000:2.0:1769074693.806525:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406528:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.806539:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406784:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.806548:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612406400:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074693.806558:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612406656:192.168.204.102@tcp:400:kworker.0 00000001:02000400:3.0:1769074693.811510:0:35664:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 8: creat open |X| close ============ 04:38:11 (1769074691) 00010000:00010000:2.0:1769074694.690265:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1769074695.010382:0:35684:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074695.010402:0:35684:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074695.010431:0:35684:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.010515:0:35684:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35684 initiator: MDT0 00010000:00010000:1.0:1769074695.010528:0:35684:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35684 initiator: MDT0 00010000:00010000:1.0:1769074695.010541:0:35684:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35684 initiator: MDT0 00000100:00100000:1.0:1769074695.010576:0:35684:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612406912:192.168.204.102@tcp:101:lfs.0 00000100:00100000:1.0:1769074695.010737:0:35684:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100542f9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074695.023163:0:35684:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612406912:192.168.204.102@tcp:101:lfs.0 00010000:00010000:1.0:1769074695.023200:0:35684:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023216:0:35684:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023230:0:35684:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023241:0:35684:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023254:0:35684:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023263:0:35684:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023271:0:35684:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074695.023282:0:35684:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134700 x1855007612406912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074711 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074695.023299:0:35684:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023314:0:35684:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074695.023322:0:35684:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074695.023358:0:35684:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074695.023366:0:35684:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:1.0:1769074695.023374:0:35684:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.023378:0:35684:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa9141837cb8 00010000:00010000:1.0:1769074695.023382:0:35684:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.023391:0:35684:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074695.025459:0:35684:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074695.025471:0:35684:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074695.025494:0:35684:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.025514:0:35684:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.025531:0:35684:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.025539:0:35684:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074695.025546:0:35684:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.025551:0:35684:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:1.0:1769074695.025559:0:35684:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa9141837cb8 00010000:00010000:1.0:1769074695.025563:0:35684:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.025572:0:35684:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.025778:0:35684:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074695.025787:0:35684:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.025795:0:35684:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.025804:0:35684:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.025810:0:35684:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074695.025814:0:35684:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.025817:0:35684:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:1.0:1769074695.025821:0:35684:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa9141837bb0 00010000:00010000:1.0:1769074695.025824:0:35684:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.025838:0:35684:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.025847:0:35684:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074695.025856:0:35684:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074695.027273:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.027305:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00800000:00000004:3.0:1769074695.027316:0:35684:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1677, max_age 1687 00000100:00100000:3.0:1769074695.027379:0:35684:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407040:192.168.204.102@tcp:41:lfs.0 00000100:00100000:3.0:1769074695.027976:0:35684:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074695.036690:0:35684:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407040:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074695.036741:0:35684:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:3.0:1769074695.036843:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036865:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036872:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036877:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036881:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036884:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036888:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036891:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036894:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036897:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036900:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036903:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036906:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036909:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036913:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036915:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036919:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036922:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036925:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036928:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036931:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036934:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036937:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036940:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036943:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036946:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036949:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036952:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036956:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036959:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036963:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036967:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036971:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036974:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036977:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036981:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036985:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036988:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036992:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.036995:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.036999:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037002:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037005:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037008:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037011:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037014:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037017:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037020:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037024:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037027:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037030:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037033:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037036:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037040:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037044:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037047:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037051:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037054:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037057:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037060:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037073:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037077:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037081:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037084:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037088:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037091:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037094:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037097:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037100:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037103:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037106:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037109:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037113:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037115:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037119:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037122:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037125:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037128:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037133:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037136:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037140:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037143:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037147:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037150:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037153:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037156:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037159:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037162:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037165:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037168:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037171:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037174:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037178:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037181:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037184:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037187:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037190:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037193:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037196:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037199:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037202:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037205:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037209:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037213:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037217:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037221:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037224:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037227:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037232:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037235:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037239:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037241:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037245:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037248:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037252:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037255:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037259:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037263:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037266:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037270:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037274:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037277:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037281:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037285:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037288:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037292:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037295:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00800000:00000080:3.0:1769074695.037299:0:35684:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.037303:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.037309:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00020000:00000004:3.0:1769074695.037319:0:35684:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1677, max_age 1687 00000100:00100000:3.0:1769074695.037368:0:35684:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407168:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074695.037470:0:35684:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074695.047261:0:35684:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407168:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074695.047303:0:35684:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074695.047406:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.047428:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00020000:00000004:3.0:1769074695.047439:0:35684:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1677, max_age 1687 00000100:00100000:3.0:1769074695.047485:0:35684:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407296:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074695.047579:0:35684:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074695.051886:0:35684:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:35684:1855007612407296:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074695.051929:0:35684:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074695.052119:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052145:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052155:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052160:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052165:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052169:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052173:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052177:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052181:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052184:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052189:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052192:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.052196:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.052199:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053128:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053151:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053167:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053173:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053177:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053181:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053185:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053188:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053191:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053194:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053198:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053201:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053862:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053874:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053879:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053882:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053886:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053889:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053893:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053897:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053901:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053905:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053909:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053912:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053915:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053919:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053922:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053925:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053928:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053931:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053934:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053937:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053941:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053943:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053946:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053949:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053952:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053955:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053969:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053974:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053978:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053982:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053986:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053989:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.053992:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.053995:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054000:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054003:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054006:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054009:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054014:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054017:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054021:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054024:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054028:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054032:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054035:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054039:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054042:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054046:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054050:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054053:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054057:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054060:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054076:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054082:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054089:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054093:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054098:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054101:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054104:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054108:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054112:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054116:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054120:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054124:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054128:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054131:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054135:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054138:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054142:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054146:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054149:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054153:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054157:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054160:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054164:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054167:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054171:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054174:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054177:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054181:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054184:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054187:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054191:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054194:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054199:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054202:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054221:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054224:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054228:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054231:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054235:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054238:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054242:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054246:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054250:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054253:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054257:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054261:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200080:3.0:1769074695.054264:0:35684:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc56e73b00 00020000:00000080:3.0:1769074695.054268:0:35684:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37800 uarg=0000000000000000 00000080:00200008:3.0:1769074695.054286:0:35684:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074695.189954:0:35691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074695.189994:0:35691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074695.190018:0:35691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190041:0:35691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190050:0:35691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074695.190072:0:35691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074695.190077:0:35691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:2.0:1769074695.190087:0:35691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa914164bbd0 00010000:00010000:2.0:1769074695.190092:0:35691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190101:0:35691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190122:0:35691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074695.190146:0:35691:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074695.190150:0:35691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074695.190156:0:35691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074695.190162:0:35691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190170:0:35691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190176:0:35691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074695.190180:0:35691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074695.190183:0:35691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:2.0:1769074695.190186:0:35691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa914164bcd0 00010000:00010000:2.0:1769074695.190190:0:35691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190196:0:35691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190503:0:35691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074695.190512:0:35691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074695.190520:0:35691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074695.190526:0:35691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190534:0:35691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190540:0:35691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074695.190544:0:35691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074695.190547:0:35691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8053b 00000080:00010000:2.0:1769074695.190550:0:35691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8053b from it ffffaa914164bdb0 00010000:00010000:2.0:1769074695.190553:0:35691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.190559:0:35691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.190566:0:35691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074695.190571:0:35691:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074695.190575:0:35691:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074695.190608:0:35691:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.193575:0:35691:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.193598:0:35691:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074695.193602:0:35691:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8053b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30639e7 expref: -99 pid: 35684 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074695.194525:0:35691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074695.194550:0:35691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58a80 x1855007612407552/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074695.194590:0:35691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:35691:1855007612407552:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074695.194755:0:35691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2a80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074695.205596:0:35691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:35691:1855007612407552:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074695.205709:0:35691:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10f:0x0](ffff8b100486db88) 00000080:00200000:2.0:1769074695.205762:0:35691:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:0.0:1769074695.258005:0:35694:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074695.258043:0:35694:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074695.258134:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35694 initiator: MDT0 00010000:00010000:0.0:1769074695.258146:0:35694:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35694 initiator: MDT0 00010000:00010000:0.0:1769074695.258157:0:35694:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35694 initiator: MDT0 00000100:00100000:0.0:1769074695.258189:0:35694:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612407680:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074695.259398:0:35694:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074695.290247:0:35694:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612407680:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074695.290288:0:35694:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290303:0:35694:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290317:0:35694:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290336:0:35694:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290348:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290356:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290363:0:35694:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074695.290376:0:35694:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006026680 x1855007612407680/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074711 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074695.290393:0:35694:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074695.290409:0:35694:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074695.290417:0:35694:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074695.298966:0:35694:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074695.298992:0:35694:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80542 00000080:00010000:1.0:1769074695.299004:0:35694:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.299008:0:35694:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80542 from it ffffaa9141723b58 00010000:00010000:1.0:1769074695.299016:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.299031:0:35694:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.299078:0:35694:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074695.299105:0:35694:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:1.0:1769074695.299111:0:35694:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:1.0:1769074695.299138:0:35694:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.300603:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35694 initiator: MDT0 00010000:00010000:1.0:1769074695.301099:0:35694:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35694 initiator: MDT0 00010000:00010000:1.0:1769074695.301120:0:35694:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35694 initiator: MDT0 00000100:00100000:1.0:1769074695.301169:0:35694:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612407808:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074695.303119:0:35694:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100542f9c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074695.313478:0:35694:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612407808:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074695.313515:0:35694:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313545:0:35694:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313555:0:35694:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313569:0:35694:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313582:0:35694:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313595:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313604:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.313613:0:35694:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074695.314172:0:35694:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134700 x1855007612407808/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074711 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074695.314187:0:35694:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314210:0:35694:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/271 (0 0) 00000080:00200000:1.0:1769074695.314219:0:35694:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074695.314250:0:35694:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x10f:0x0], updating i_size 0 00000080:00200000:1.0:1769074695.314254:0:35694:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x10f:0x0](ffff8b100486db88): rc = 0 00000080:00200000:1.0:1769074695.314277:0:35694:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x10f:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1769074695.314284:0:35694:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10f:0x0](ffff8b100486db88) for lock 0x68a7113811d80549 00000080:00010000:1.0:1769074695.314297:0:35694:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10f:0x0](ffff8b100486db88) 00000080:00010000:1.0:1769074695.314300:0:35694:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80549 from it ffffaa9141723b20 00010000:00010000:1.0:1769074695.314304:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314312:0:35694:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074695.314366:0:35694:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x10f:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074695.314375:0:35694:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10f:0x0](ffff8b100486db88),name=fsa-oleg402-client.virtnet 00000002:00010000:1.0:1769074695.314402:0:35694:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x10f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.314413:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314425:0:35694:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.314432:0:35694:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074695.314436:0:35694:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10f:0x0](ffff8b100486db88) 00000080:00010000:1.0:1769074695.314440:0:35694:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10f:0x0](ffff8b100486db88) for lock 0x68a7113811d80549 00000080:00010000:1.0:1769074695.314444:0:35694:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80549 from it ffffaa9141723cb8 00010000:00010000:1.0:1769074695.314448:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314455:0:35694:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.314531:0:35694:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074695.314539:0:35694:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.314546:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314553:0:35694:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.314559:0:35694:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074695.314563:0:35694:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.314565:0:35694:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80542 00000080:00010000:1.0:1769074695.314569:0:35694:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80542 from it ffffaa9141723c10 00010000:00010000:1.0:1769074695.314571:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314577:0:35694:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.314585:0:35694:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074695.314594:0:35694:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:1.0:1769074695.314599:0:35694:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074695.314605:0:35694:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074695.314610:0:35694:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.314617:0:35694:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.314635:0:35694:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074695.321229:0:35694:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074695.321245:0:35694:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80542 00000080:00010000:1.0:1769074695.321259:0:35694:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80542 from it ffffaa9141723d68 00010000:00010000:1.0:1769074695.321267:0:35694:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074695.321282:0:35694:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074695.321305:0:35694:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074695.321522:0:35694:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074695.321586:0:35694:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.322051:0:35694:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074695.322080:0:35694:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x10f:0x0]: losing layout lock 00010000:00010000:2.0:1769074695.322145:0:35694:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.322153:0:35694:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.322160:0:35694:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074695.322163:0:35694:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9e00/0x68a7113811d80549 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063a1f expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074695.322188:0:35694:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac800/0x68a7113811d80542 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063a03 expref: -99 pid: 35694 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074695.322306:0:35694:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074695.322324:0:35694:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59180 x1855007612408064/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074695.322357:0:35694:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612408064:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074695.322466:0:35694:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2a80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074695.340733:0:35694:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:35694:1855007612408064:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074695.340822:0:35694:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x10f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074695.340837:0:35694:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x10f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074695.340847:0:35694:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10f:0x0](ffff8b100486db88) 00010000:00010000:3.0:1769074695.718465:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074696.736000:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074697.760502:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074698.784086:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074698.917225:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612408192:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.917334:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.917379:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074698.917386:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408320:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.917412:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.924314:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408320:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.924332:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.924342:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612408192:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074698.924353:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612408448:192.168.204.102@tcp:400:kworker.0 00000001:02000400:1.0:1769074699.078618:0:35725:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:3.0:1769074699.218934:0:35727:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074699.218975:0:35727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074699.219080:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35727 initiator: MDT0 00010000:00010000:3.0:1769074699.219093:0:35727:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00010000:00010000:3.0:1769074699.219105:0:35727:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00000100:00100000:3.0:1769074699.219137:0:35727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612408704:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074699.219256:0:35727:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074699.224234:0:35727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612408704:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074699.224273:0:35727:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224289:0:35727:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224304:0:35727:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224317:0:35727:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224329:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224337:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224345:0:35727:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074699.224358:0:35727:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037dfb80 x1855007612408704/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074715 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074699.224376:0:35727:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224391:0:35727:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074699.224399:0:35727:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074699.224438:0:35727:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074699.224447:0:35727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80550 00000080:00010000:3.0:1769074699.224459:0:35727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074699.224463:0:35727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80550 from it ffffaa914176bbd0 00010000:00010000:3.0:1769074699.224467:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224475:0:35727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.224518:0:35727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074699.224549:0:35727:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:3.0:1769074699.224554:0:35727:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074699.224572:0:35727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074699.224581:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224592:0:35727:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.224598:0:35727:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074699.224602:0:35727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074699.224606:0:35727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80550 00000080:00010000:3.0:1769074699.224610:0:35727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80550 from it ffffaa914176bcd0 00010000:00010000:3.0:1769074699.224613:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224621:0:35727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.224629:0:35727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074699.224865:0:35727:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074699.224876:0:35727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074699.224883:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224892:0:35727:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.224898:0:35727:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074699.224903:0:35727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074699.224906:0:35727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80550 00000080:00010000:3.0:1769074699.224910:0:35727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80550 from it ffffaa914176bdb0 00010000:00010000:3.0:1769074699.224913:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.224920:0:35727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.224929:0:35727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074699.224935:0:35727:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:3.0:1769074699.224939:0:35727:0:(namei.c:2020:ll_mknod()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:3.0:1769074699.224972:0:35727:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.225046:0:35727:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.225054:0:35727:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074699.225058:0:35727:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80550 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063a49 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074699.225089:0:35727:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074699.225099:0:35727:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dfb80 x1855007612408960/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074699.225126:0:35727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612408960:192.168.204.102@tcp:36:multiop.0 00000100:00100000:3.0:1769074699.225267:0:35727:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074699.253209:0:35727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612408960:192.168.204.102@tcp:36:multiop.0 00000080:00200000:3.0:1769074699.253277:0:35727:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00200000:3.0:1769074699.253331:0:35727:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f8.replay-single, unhashed 1 00000080:00200000:3.0:1769074699.253594:0:35727:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074699.253628:0:35727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074699.253757:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35727 initiator: MDT0 00010000:00010000:3.0:1769074699.253768:0:35727:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00010000:00010000:3.0:1769074699.253779:0:35727:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00000100:00100000:3.0:1769074699.253805:0:35727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612409088:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074699.253887:0:35727:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074699.258532:0:35727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612409088:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074699.258569:0:35727:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258584:0:35727:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258598:0:35727:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258613:0:35727:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258911:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258929:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258938:0:35727:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074699.258951:0:35727:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037ddf80 x1855007612409088/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074715 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074699.258967:0:35727:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.258983:0:35727:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074699.258992:0:35727:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074699.259041:0:35727:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074699.259049:0:35727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80557 00000080:00010000:3.0:1769074699.259058:0:35727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074699.259061:0:35727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80557 from it ffffaa914176bbf8 00010000:00010000:3.0:1769074699.259065:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.259074:0:35727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074699.259104:0:35727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074699.259128:0:35727:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1002fc7980, open_flags 8000, mode 0 opened 0 00000080:00200000:3.0:1769074699.259137:0:35727:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:3.0:1769074699.259165:0:35727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f8.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:3.0:1769074699.260190:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35727 initiator: MDT0 00010000:00010000:3.0:1769074699.260206:0:35727:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00000020:00100000:3.0:1769074699.260222:0:35727:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074699.260234:0:35727:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037ddf80 x1855007612409344/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074699.260245:0:35727:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35727 initiator: MDT0 00000100:00100000:3.0:1769074699.260270:0:35727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612409344:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074699.262124:0:35727:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074699.285048:0:35727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612409344:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074699.285085:0:35727:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285123:0:35727:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285133:0:35727:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285147:0:35727:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285161:0:35727:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285175:0:35727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285183:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285190:0:35727:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074699.285204:0:35727:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074699.285219:0:35727:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074699.285237:0:35727:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074699.285248:0:35727:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/272 (0 0) 00000080:00200000:3.0:1769074699.285255:0:35727:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074699.285282:0:35727:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x110:0x0], updating i_size 0 00000080:00200000:3.0:1769074699.285285:0:35727:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x110:0x0](ffff8b100485aa08): rc = 0 00000080:00200000:2.0:1769074699.285610:0:35727:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x110:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1769074699.285620:0:35727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d8055e 00000080:00010000:2.0:1769074699.285635:0:35727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00200008:2.0:1769074699.285680:0:35727:0:(file.c:1001:ll_file_open()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), flags 100000 00000002:00100000:2.0:1769074699.285710:0:35727:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074715 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1769074699.285783:0:35727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8055e from it ffff8b10065d7540 00010000:00010000:2.0:1769074699.285788:0:35727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074699.285798:0:35727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074699.285828:0:35727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08), inode mode 81a4 mask 44 00010000:00010000:3.0:1769074699.810231:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074700.831961:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074701.858120:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074702.879824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074703.909365:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074704.035354:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074704.035482:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074704.035531:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074704.035548:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612409472:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074704.035587:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409728:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074704.044958:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409728:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074704.044987:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409856:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074704.935270:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074705.953544:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074706.976239:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074708.001921:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074709.025218:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074709.154195:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074709.154300:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410368:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074709.154336:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074709.154343:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612409984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074709.154374:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074709.158198:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074709.158218:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410368:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074710.048021:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074711.072089:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074712.098235:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074713.119900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074714.143953:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074714.273308:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612410496:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074714.273398:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074714.273433:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074714.273440:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410624:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074714.273463:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410880:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074714.296248:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410880:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074714.296268:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410752:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074715.172911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074716.191905:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074717.216412:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074718.245921:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074719.263876:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074719.391879:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:2.0:1769074719.391889:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b101127b800: disconnect after 24s idle 00000100:00080000:2.0:1769074719.391915:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:2.0:1769074719.391919:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b101127b800: disconnect after 24s idle 00000100:00100000:2.0:1769074719.391978:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612411008:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074719.392118:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074719.392127:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612411136:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074719.392376:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411264:192.168.204.102@tcp:9:kworker.0 00000100:00100000:1.0:1769074719.392415:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411392:192.168.204.102@tcp:9:kworker.0 00000100:00080000:0.0:1769074719.401829:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1005d5bb80 x1855007612411264/t0(0) o9->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074729 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074719.401848:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1769074719.401854:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411264:192.168.204.102@tcp:9:kworker.0 00000100:00080000:0.0:1769074719.401872:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1005d5b800 x1855007612411392/t0(0) o9->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074729 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074719.401881:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:0.0:1769074719.401885:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411392:192.168.204.102@tcp:9:kworker.0 00010000:00010000:3.0:1769074720.288626:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074720.417774:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074704/real 1769074704] req@ffff8b1005d5b480 x1855007612409472/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074720 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074720.417819:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d5b480 x1855007612409472/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074720 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074720.417832:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1769074720.417836:0:2367:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1769074720.417841:0:2367:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1769074720.417844:0:2367:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1769074720.417850:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5b480 x1855007612409472/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074720 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074720.417860:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5aa00 x1855007612409984/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074725 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074720.417869:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006134700 x1855007612410496/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074730 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074720.417894:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5b100 x1855007612411008/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074735 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1769074720.417907:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1769074720.417911:0:2367:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1769074720.417914:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1769074720.417917:0:2367:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:2.0:1769074720.417951:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074704/real 1769074704] req@ffff8b1005d58700 x1855007612409600/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074720 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074720.417976:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58700 x1855007612409600/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074720 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074720.417984:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1769074720.417988:0:2367:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1769074720.417993:0:2367:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1769074720.418017:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612409984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074720.418026:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612409472:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074720.418034:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612409600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074720.418091:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612411008:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074720.418291:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612410496:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074720.420894:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074720.420909:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074720.420914:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074720.420926:0:38:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1652 00000100:00080000:0.0:1769074720.420948:0:38:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769074720.421009:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612411520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074720.421047:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074720.421051:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074720.421053:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074720.421058:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1666 00000100:00080000:0.0:1769074720.421064:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074720.421077:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074720.421090:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612411648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074720.421137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612411520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769074720.421518:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411648:192.168.204.102@tcp:38:kworker.0 00010000:00010000:0.0:1769074721.313179:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074722.336959:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074723.361757:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074724.384313:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074724.447838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074724.447855:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769074724.447860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074724.447863:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1769074725.408305:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074725.471839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074725.471855:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769074725.471859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074725.471865:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:3.0:1769074725.535993:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074709/real 1769074709] req@ffff8b1005d5a680 x1855007612410112/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074725 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074725.536041:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d5a680 x1855007612410112/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074725 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074725.536051:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 56, was 55): CONNECTING 00000100:00100000:3.0:1769074725.536073:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612410112:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074726.432285:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074727.456071:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074728.481582:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074729.505592:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074730.528086:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074730.655926:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074720/real 1769074720] req@ffff8b1006026a00 x1855007612411520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1769074730.655927:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074714/real 1769074714] req@ffff8b1006134a80 x1855007612410624/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074730.656015:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006134a80 x1855007612410624/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074730.656030:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 56, was 55): CONNECTING 00000100:00100000:3.0:1769074730.656031:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612411520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074730.656053:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074720/real 1769074720] req@ffff8b1006026300 x1855007612411648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074730.656064:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612410624:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074730.656074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612411648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074730 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.656097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074730.656126:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769074730.656130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612411520:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074730.656136:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074730.656146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769074730.656153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411648:192.168.204.102@tcp:38:kworker.0 00000100:00080000:1.0:1769074730.656373:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074730.656380:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769074730.656384:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769074730.656391:0:38:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1713 00000100:00080000:1.0:1769074730.656402:0:38:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1769074730.656408:0:38:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769074730.656451:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612411776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769074730.656483:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074730.656487:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769074730.656489:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769074730.656494:0:38:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1713 00000100:00080000:1.0:1769074730.656497:0:38:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1769074730.656501:0:38:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769074730.656510:0:38:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769074730.656520:0:38:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612411904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074730.656626:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612411776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769074730.656778:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411904:192.168.204.102@tcp:38:kworker.0 00010000:00080000:3.0:1769074730.663323:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1006136a00 x1855007612411904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074745 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074730.663354:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378632 gen 1 00000100:00100000:3.0:1769074730.663387:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074715 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.663405:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 20 00000100:00080000:3.0:1769074730.663438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1769074730.663446:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074730.663448:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378632 gen 1 00000100:00100000:3.0:1769074730.663453:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074715 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.663460:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 81604378632 last 0 00000100:00080000:3.0:1769074730.663476:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074715 ref 1 fl New:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074730.663491:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612411904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769074730.663524:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612408960:192.168.204.102@tcp:36:multiop.0 00000100:00080000:3.0:1769074730.673485:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:3.0:1769074730.673505:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1769074730.673515:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf30635a3 to 0x143201ebf3063c2c 00000100:00080000:3.0:1769074730.673520:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1769074730.673526:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:0.0:1769074730.675718:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612411776:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074730.676002:0:35988:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:3.0:1769074730.676008:0:35988:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1769074730.676013:0:35988:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1769074730.676031:0:35988:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074730.676042:0:35988:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074730.676049:0:35988:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074730.676073:0:35988:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074730.676085:0:35988:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d804f5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf306368a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074730.676424:0:35988:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074730.676432:0:35988:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074730.676438:0:35988:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074730.676443:0:35988:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074730.676447:0:35988:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80503 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf30636a6 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074730.676461:0:35988:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074730.676479:0:35988:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074730.676485:0:35988:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074730.676488:0:35988:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074730.676492:0:35988:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d8050a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063732 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074730.676504:0:35988:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074730.676510:0:35988:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074730.676515:0:35988:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1769074730.676518:0:35988:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074730.676521:0:35988:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d804fc lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf306369f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:3.0:1769074730.676539:0:35988:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1769074730.676545:0:35988:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1769074730.676548:0:35988:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1769074730.676551:0:35988:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1769074730.676556:0:35988:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:3.0:1769074730.744049:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074746 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.744107:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074730.744111:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378632 gen 1 00000100:00100000:3.0:1769074730.744117:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074746 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.744125:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 81604378632 last 81604378633 00000100:00080000:3.0:1769074730.744141:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074715 ref 1 fl New:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1769074730.744154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612408960:192.168.204.102@tcp:36:multiop.0 00000100:00100000:3.0:1769074730.744175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612409344:192.168.204.102@tcp:101:multiop.0 00000100:00080000:3.0:1769074730.756072:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074746 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00080000:3.0:1769074730.756096:0:2363:0:(mdc_request.c:695:mdc_replay_open()) updating handle from 0x143201ebf3063a88 to 0x143201ebf3063c48 00000100:00020000:3.0:1769074730.756101:0:2363:0:(client.c:3418:ptlrpc_replay_interpret()) @@@ status 301, old was 0 req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074746 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1769074730.792068:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074730.792077:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378632 gen 1 00000100:00100000:3.0:1769074730.798755:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b10037dfb80 x1855007612408960/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074746 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074730.798775:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 81604378632 last 81604378634 00000100:00080000:3.0:1769074730.798782:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:0.0:1769074730.799739:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612409344:192.168.204.102@tcp:101:multiop.0 00010000:00080000:3.0:1769074730.799806:0:35989:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:3.0:1769074730.799811:0:35989:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (2) 00000080:00200000:3.0:1769074730.799845:0:35989:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x110:0x0]: losing layout lock 00010000:00010000:3.0:1769074730.799867:0:35989:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d8055e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3063a81 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074730.799895:0:35989:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d80557 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3063a65 expref: -99 pid: 35727 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074730.799905:0:35989:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:3.0:1769074730.800326:0:35989:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:0.0:1769074730.800406:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412032:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00080000:2.0:1769074730.902721:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:2.0:1769074730.902739:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b10037ded80 x1855007612412032/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074746 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074730.902749:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b10037ded80 x1855007612412032/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074746 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074730.902756:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:2.0:1769074730.903778:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00100000:2.0:1769074730.903800:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412032:192.168.204.102@tcp:400:ldlm_lock_repla.0 00010000:00010000:2.0:1769074731.551818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074732.159522:0:36016:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074732.159540:0:36016:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074732.159569:0:36016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074732.159682:0:36016:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 36016 initiator: MDT0 00010000:00010000:2.0:1769074732.159693:0:36016:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36016 initiator: MDT0 00010000:00010000:2.0:1769074732.159704:0:36016:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36016 initiator: MDT0 00000100:00100000:2.0:1769074732.159731:0:36016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412160:192.168.204.102@tcp:101:lfs.0 00000100:00100000:2.0:1769074732.159831:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074732.164832:0:36016:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 81604378634 gen 1 00000100:00100000:2.0:1769074732.164861:0:36016:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074746 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074732.164876:0:36016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412160:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074732.164895:0:36016:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164906:0:36016:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164919:0:36016:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164930:0:36016:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164942:0:36016:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164950:0:36016:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164958:0:36016:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074732.164966:0:36016:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5b100 x1855007612412160/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074748 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074732.164979:0:36016:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.164996:0:36016:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074732.165003:0:36016:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074732.165029:0:36016:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074732.165035:0:36016:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074732.165041:0:36016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074732.165045:0:36016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa914205fcb8 00010000:00010000:2.0:1769074732.165048:0:36016:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.165056:0:36016:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074732.165789:0:36016:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074732.165797:0:36016:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074732.165813:0:36016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074732.165824:0:36016:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.165836:0:36016:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074732.165843:0:36016:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074732.165848:0:36016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074732.165852:0:36016:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074732.165856:0:36016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa914205fcb8 00010000:00010000:2.0:1769074732.165860:0:36016:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.165871:0:36016:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074732.165985:0:36016:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074732.165994:0:36016:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074732.166000:0:36016:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.166008:0:36016:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074732.166013:0:36016:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074732.166016:0:36016:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074732.166018:0:36016:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074732.166022:0:36016:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa914205fbb0 00010000:00010000:2.0:1769074732.166024:0:36016:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.166031:0:36016:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074732.166037:0:36016:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074732.166044:0:36016:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074732.166134:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.166147:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00800000:00000004:2.0:1769074732.166154:0:36016:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1688, max_age 1724 00000100:00100000:2.0:1769074732.166193:0:36016:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412288:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074732.166271:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074732.170553:0:36016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412288:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074732.170583:0:36016:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:2.0:1769074732.170702:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170718:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170725:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170729:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170734:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170737:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170741:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170744:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170747:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170750:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170753:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.170756:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.170759:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171181:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171191:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171195:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171200:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171203:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171207:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171210:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171213:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171217:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171221:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171224:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171228:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171232:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171236:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171239:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171244:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171247:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171251:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171254:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171258:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171262:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171266:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171269:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171272:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171275:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171278:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171282:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171286:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171289:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171293:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171296:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171300:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171302:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171306:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171309:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171312:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171315:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171318:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171321:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171325:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171327:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171331:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171334:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171337:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171340:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171343:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171346:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171350:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171353:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171357:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171359:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171363:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171365:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171368:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171371:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171373:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171376:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171379:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171382:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171385:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171388:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171391:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171394:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171396:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171399:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171401:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171404:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171407:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171409:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171412:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171414:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171417:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171420:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171423:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171426:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171429:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171432:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171435:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171437:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171449:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171455:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171460:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171463:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171467:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171471:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171474:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171477:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171481:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171484:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171488:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171491:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171495:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171498:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171501:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171505:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171509:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171513:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171516:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171519:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171524:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171527:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171530:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171534:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171538:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171541:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171545:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171548:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171552:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171555:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171559:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171563:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171566:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171570:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171574:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00800000:00000080:2.0:1769074732.171577:0:36016:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00000080:00200080:2.0:1769074732.171580:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:2.0:1769074732.171586:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00020000:00000004:2.0:1769074732.171593:0:36016:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1688, max_age 1724 00000100:00080000:2.0:1769074732.171619:0:36016:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1666 00000100:00080000:2.0:1769074732.171632:0:36016:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074732.171659:0:36016:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:2.0:1769074732.171684:0:36016:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58700 x1855007612412416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074732.171727:0:36016:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d5b100 x1855007612412544/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074732.171737:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 0 seconds 00000100:00100000:3.0:1769074732.172463:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412416:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074732.178102:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934747 gen 8 00000100:00080000:3.0:1769074732.178127:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b101127b800: connect to target with instance 2 00000100:00080000:3.0:1769074732.178141:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:3.0:1769074732.178157:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074732.178167:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074732.178174:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1769074732.178180:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:3.0:1769074732.178189:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:3.0:1769074732.178190:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:3.0:1769074732.178206:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000003e0d4df8): req@ffff8b1005d5b100 x1855007612412544/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074732.178229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412416:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074732.178378:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074732.189025:0:36016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412544:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074732.189077:0:36016:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074732.189168:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:2.0:1769074732.189191:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab800 uarg=0000000000000000 00020000:00000004:2.0:1769074732.189199:0:36016:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1688, max_age 1724 00000100:00080000:2.0:1769074732.189214:0:36016:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1666 00000100:00080000:2.0:1769074732.189228:0:36016:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074732.189241:0:36016:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:2.0:1769074732.189269:0:36016:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612412672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074732.189319:0:36016:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d5b100 x1855007612412800/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074732.189329:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 0 seconds 00000100:00100000:3.0:1769074732.189702:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412672:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074732.196553:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967450 gen 8 00000100:00080000:2.0:1769074732.196584:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b101127b800: connect to target with instance 1 00000100:00080000:2.0:1769074732.196600:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1769074732.196618:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074732.196628:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074732.196634:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:2.0:1769074732.196680:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1769074732.196691:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074732.196693:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074732.196710:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 000000003e0d4df8): req@ffff8b1005d5b100 x1855007612412800/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074732.196738:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612412672:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074732.197005:0:36016:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f900 going to sleep for 11 seconds 00000100:00100000:3.0:1769074732.201588:0:36016:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36016:1855007612412800:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074732.201631:0:36016:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074732.201761:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201784:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201792:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201797:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201801:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201805:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201809:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201812:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201815:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201818:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201822:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201825:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201829:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201833:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201836:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201840:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201844:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201849:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201853:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201856:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201860:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201863:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201867:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201870:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201873:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201876:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201879:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201883:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201886:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201889:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201892:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201895:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201898:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201901:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201904:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201907:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201911:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201914:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201918:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201921:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201925:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201927:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201932:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201936:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201939:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201942:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201946:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201950:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201954:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201957:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201961:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201965:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201969:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201972:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201976:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201978:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201982:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201985:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201988:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201992:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.201995:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.201998:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202002:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202006:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202010:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202014:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202017:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202021:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202025:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202030:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202034:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202037:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202041:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202045:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202048:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202051:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202055:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202058:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202061:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202077:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202083:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202087:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202091:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202095:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202099:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202103:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202107:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202111:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202114:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202117:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202121:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202124:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202128:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202131:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202134:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202138:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202142:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202145:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202150:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202154:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202158:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202162:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202166:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202170:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202173:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202176:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202180:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202183:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202188:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202191:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202196:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202199:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202205:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202208:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202212:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202215:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202218:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202222:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202226:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202229:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202233:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202236:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202240:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202243:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200080:3.0:1769074732.202246:0:36016:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffd4c5f7ca0 00020000:00000080:3.0:1769074732.202250:0:36016:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f34c00 uarg=0000000000000000 00000080:00200008:3.0:1769074732.202271:0:36016:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 10000000:01000000:2.0:1769074732.584864:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074732.584874:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 25 10000000:01000000:2.0:1769074732.584882:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074732.584960:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.584975:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.584988:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074732.585025:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612412928:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074732.585145:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00010000:00010000:3.0:1769074732.585751:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074732.610052:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612412928:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074732.610100:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.610116:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074732.610245:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413056:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074732.610360:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337fa80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074732.638909:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413056:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1769074732.638984:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1769074732.639185:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074732.639200:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074732.639215:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074732.639225:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1769074732.639228:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:3.0:1769074732.639232:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1769074732.639276:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074732.639284:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074732.639299:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074732.639325:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413184:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074732.639431:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074732.663269:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413184:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1769074732.663323:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074732.663347:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074732.663511:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413312:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1769074732.663682:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5cad80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074732.674416:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413312:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074732.674529:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413440:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074732.674688:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074732.688633:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413440:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1769074732.690249:0:36065:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:2.0:1769074732.690357:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:2.0:1769074732.690384:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1769074732.690400:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.690418:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074732.690431:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1769074732.690443:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:2.0:1769074732.690448:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1769074732.690516:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.690527:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.690540:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074732.690583:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413568:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074732.690752:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f480 going to sleep for 11 seconds 00000100:00100000:2.0:1769074732.711620:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413568:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074732.711675:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074732.711692:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074732.711799:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413696:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074732.711902:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100337f480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074732.720792:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413696:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074732.720890:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413824:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074732.722513:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ece40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074732.745399:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413824:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1769074732.746775:0:36066:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1769074732.747402:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1769074732.747444:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1769074732.747468:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074732.747485:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074732.747504:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1769074732.747507:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:0.0:1769074732.747510:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1769074732.747566:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074732.747575:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074732.747587:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074732.747621:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413952:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074732.747767:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a3e3c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074732.757974:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612413952:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074732.758174:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074732.758192:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074732.767866:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612414080:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1769074732.768012:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a3ed80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074732.778349:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612414080:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:1.0:1769074732.778414:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:1.0:1769074732.778428:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074732.778452:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074733.603386:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074734.629747:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074735.648916:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074735.776301:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.776446:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.776485:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074735.776492:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612414208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.776526:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414464:192.168.204.102@tcp:400:kworker.0 00000100:00000400:3.0:1769074735.776993:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074719/real 1769074719] req@ffff8b1005d5a300 x1855007612411136/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074735 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074735.777024:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d5a300 x1855007612411136/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074735 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074735.777041:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612411136:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.783632:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.783697:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.783711:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612414592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074735.792041:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612414208:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074736.671890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1769074737.548179:0:36271:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:1.0:1769074737.712928:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074738.721178:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074739.085521:0:36425:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:2.0:1769074739.747492:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074740.767865:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074740.897770:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612414720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.897869:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612414976:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.897902:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074740.897908:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612414848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.897928:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612415104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.903896:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612414720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.905609:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612414848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.914608:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612415104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074740.914670:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612414976:192.168.204.102@tcp:400:kworker.0 00000080:00200008:2.0:1769074741.398046:0:36464:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074741.398062:0:36464:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.398087:0:36464:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.398107:0:36464:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.398123:0:36464:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.398131:0:36464:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.398139:0:36464:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.398143:0:36464:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.398149:0:36464:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9140c87cb8 00010000:00010000:2.0:1769074741.398154:0:36464:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.398162:0:36464:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074741.399439:0:36464:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074741.399448:0:36464:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.399466:0:36464:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.399484:0:36464:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.399499:0:36464:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.399507:0:36464:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.399512:0:36464:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.399515:0:36464:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.399521:0:36464:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9140c87cb8 00010000:00010000:2.0:1769074741.399525:0:36464:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.399533:0:36464:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.399796:0:36464:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.399805:0:36464:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.399812:0:36464:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.399821:0:36464:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.399826:0:36464:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.399830:0:36464:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.399833:0:36464:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.399837:0:36464:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9140c87bb0 00010000:00010000:2.0:1769074741.399840:0:36464:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.399845:0:36464:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.399853:0:36464:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074741.399861:0:36464:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074741.400020:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.400039:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00800000:00000004:2.0:1769074741.400048:0:36464:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1725, max_age 1733 00000100:00100000:2.0:1769074741.400106:0:36464:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415232:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074741.400360:0:36464:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074741.422448:0:36464:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415232:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074741.422485:0:36464:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:2.0:1769074741.422576:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422594:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422602:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422607:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422611:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422615:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422619:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422622:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422626:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422629:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422633:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422674:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422682:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422686:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422691:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422695:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422699:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422703:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422707:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422710:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422714:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422718:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422722:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422726:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422730:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422734:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422739:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422742:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422746:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422748:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422752:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422755:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422758:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422761:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422764:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422768:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422771:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422774:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422777:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422780:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422783:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422786:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422789:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422792:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422796:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422798:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422802:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422805:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422809:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422811:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422815:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422818:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422821:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422824:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422827:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422831:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422837:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422841:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422847:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422850:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422854:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422857:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422861:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422864:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422867:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422870:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422874:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422877:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422881:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422883:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422886:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422890:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422893:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422896:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422900:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422903:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422906:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422909:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422912:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422915:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422918:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422921:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422924:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422927:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422931:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422934:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422938:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422942:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422945:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422948:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422951:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422954:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422957:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422961:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422964:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422967:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422972:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422976:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422980:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422983:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422987:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422989:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.422993:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.422996:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423000:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423004:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423007:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423011:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423014:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423016:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423020:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423023:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423027:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423030:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423034:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423038:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423042:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423047:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423051:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423054:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423058:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423061:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423065:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423068:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423072:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423075:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423078:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00800000:00000080:2.0:1769074741.423081:0:36464:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.423085:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.423090:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00020000:00000004:2.0:1769074741.423097:0:36464:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1725, max_age 1733 00000100:00100000:2.0:1769074741.423133:0:36464:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415360:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074741.424735:0:36464:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074741.429797:0:36464:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415360:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074741.429827:0:36464:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074741.430448:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.430474:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00020000:00000004:2.0:1769074741.430483:0:36464:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1725, max_age 1733 00000100:00100000:2.0:1769074741.430534:0:36464:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415488:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074741.430631:0:36464:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074741.436052:0:36464:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:36464:1855007612415488:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074741.436086:0:36464:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074741.436311:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436329:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436338:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436343:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436346:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436350:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436355:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436358:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436362:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436366:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436370:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436373:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436377:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436380:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436384:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436388:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436391:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436395:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436398:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436401:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436405:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436409:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436412:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436416:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436420:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436424:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436428:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436431:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436436:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436440:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436444:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436447:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436451:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436455:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436458:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436461:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436466:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436468:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436472:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436475:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436479:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436481:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436485:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436488:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436492:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436495:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436499:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436503:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436507:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436509:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436513:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436516:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436520:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436524:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436527:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436530:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436534:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436537:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436541:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436544:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436548:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436551:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436555:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436558:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436561:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436564:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436568:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436571:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436575:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436578:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436582:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436585:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436588:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436591:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436593:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436596:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436600:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436602:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436606:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436610:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436613:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436616:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436620:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436623:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436627:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436630:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436633:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436636:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436681:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436686:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436690:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436693:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436700:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436704:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436711:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436713:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436716:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436719:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436722:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436725:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436729:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436732:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436735:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436738:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436741:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436744:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436747:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436750:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436753:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436756:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436761:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436764:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436767:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436770:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436773:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436776:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436780:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436784:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436787:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436791:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436794:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436797:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436800:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436803:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074741.436807:0:36464:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff26347a80 00020000:00000080:2.0:1769074741.436809:0:36464:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200008:2.0:1769074741.436824:0:36464:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:0.0:1769074741.525551:0:36480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074741.525579:0:36480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074741.525596:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074741.525615:0:36480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074741.525622:0:36480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074741.525629:0:36480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074741.525635:0:36480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:0.0:1769074741.526029:0:36480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9141463b58 00010000:00010000:0.0:1769074741.526034:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074741.526043:0:36480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074741.526058:0:36480:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074741.526077:0:36480:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=9 00000080:00200000:0.0:1769074741.526082:0:36480:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:0.0:1769074741.526096:0:36480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f8.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074741.526141:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 36480 initiator: MDT0 00010000:00010000:0.0:1769074741.526148:0:36480:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36480 initiator: MDT0 00010000:00010000:0.0:1769074741.526157:0:36480:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36480 initiator: MDT0 00000100:00100000:0.0:1769074741.526577:0:36480:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:36480:1855007612415616:192.168.204.102@tcp:101:ls.0 00000100:00100000:0.0:1769074741.526693:0:36480:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074741.545888:0:36480:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:36480:1855007612415616:192.168.204.102@tcp:101:ls.0 00010000:00010000:3.0:1769074741.545923:0:36480:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.545945:0:36480:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.545954:0:36480:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.545972:0:36480:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.545984:0:36480:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.545995:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546003:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546010:0:36480:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074741.546020:0:36480:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024000 x1855007612415616/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074757 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074741.546033:0:36480:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546041:0:36480:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/272 (0 0) 00000080:00200000:3.0:1769074741.546048:0:36480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074741.546066:0:36480:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x110:0x0], updating i_size 0 00000080:00200000:3.0:1769074741.546070:0:36480:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x110:0x0](ffff8b100485aa08): rc = 0 00000080:00200000:3.0:1769074741.546083:0:36480:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x110:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074741.546088:0:36480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d80588 00000080:00010000:3.0:1769074741.546101:0:36480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00010000:3.0:1769074741.546105:0:36480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80588 from it ffffaa9141463b20 00010000:00010000:3.0:1769074741.546109:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546117:0:36480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074741.546157:0:36480:0:(file.c:6252:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), request_mask 2, flags 0, foreign 0 00000080:00200000:3.0:1769074741.546162:0:36480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08),name=f8.replay-single 00000002:00010000:3.0:1769074741.546192:0:36480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074741.546200:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546210:0:36480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074741.546218:0:36480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074741.546223:0:36480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00010000:3.0:1769074741.546227:0:36480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d80588 00000080:00010000:3.0:1769074741.546231:0:36480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80588 from it ffffaa9141463cb8 00010000:00010000:3.0:1769074741.546234:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546241:0:36480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074741.546276:0:36480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074741.546283:0:36480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074741.546300:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546309:0:36480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074741.546316:0:36480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074741.546320:0:36480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074741.546322:0:36480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:3.0:1769074741.546326:0:36480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9141463b58 00010000:00010000:3.0:1769074741.546329:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546336:0:36480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074741.546344:0:36480:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074741.546349:0:36480:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:3.0:1769074741.546354:0:36480:0:(file.c:6252:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), request_mask 2, flags 0, foreign 0 00000080:00200000:3.0:1769074741.546358:0:36480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08),name=f8.replay-single 00000002:00010000:3.0:1769074741.546365:0:36480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074741.546371:0:36480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546378:0:36480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074741.546384:0:36480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074741.546388:0:36480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00010000:3.0:1769074741.546391:0:36480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d80588 00000080:00010000:3.0:1769074741.546394:0:36480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80588 from it ffffaa9141463cb8 00010000:00010000:3.0:1769074741.546397:0:36480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074741.546404:0:36480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074741.551376:0:36483:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074741.551406:0:36483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074741.551423:0:36483:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074741.551439:0:36483:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074741.551447:0:36483:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074741.551455:0:36483:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074741.551459:0:36483:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:1.0:1769074741.551465:0:36483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa91415bbb58 00010000:00010000:1.0:1769074741.551469:0:36483:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074741.551478:0:36483:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074741.551495:0:36483:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074741.551502:0:36483:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:1.0:1769074741.551510:0:36483:0:(file.c:6252:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074741.551515:0:36483:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08),name=f8.replay-single 00000002:00010000:1.0:1769074741.551525:0:36483:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074741.551533:0:36483:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074741.551546:0:36483:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074741.551552:0:36483:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074741.551557:0:36483:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00010000:1.0:1769074741.551559:0:36483:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d80588 00000080:00010000:1.0:1769074741.551563:0:36483:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80588 from it ffffaa91415bbcb8 00010000:00010000:1.0:1769074741.551566:0:36483:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074741.551574:0:36483:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074741.556743:0:35727:0:(file.c:424:ll_file_release()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), flags 100000 00000002:00100000:2.0:1769074741.556819:0:35727:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074746 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074741.556833:0:35727:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074741.556838:0:35727:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58000 x1855007612415872/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074741.556881:0:35727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612415872:192.168.204.102@tcp:35:multiop.0 00000100:00100000:2.0:1769074741.557280:0:35727:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c39c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074741.561972:0:35727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:35727:1855007612415872:192.168.204.102@tcp:35:multiop.0 00000002:00100000:3.0:1769074741.562004:0:35727:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10037ddf80 x1855007612409344/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074746 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074741.587416:0:36484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.587446:0:36484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.587463:0:36484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.587479:0:36484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.587487:0:36484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.587496:0:36484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.587500:0:36484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.587507:0:36484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9141543b58 00010000:00010000:2.0:1769074741.587511:0:36484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.587520:0:36484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.587537:0:36484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074741.587546:0:36484:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:2.0:1769074741.587554:0:36484:0:(file.c:6252:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffff8b100485aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074741.587559:0:36484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08),name=f8.replay-single 00000002:00010000:2.0:1769074741.587572:0:36484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.587578:0:36484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.587586:0:36484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.587592:0:36484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.587596:0:36484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) 00000080:00010000:2.0:1769074741.587599:0:36484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffff8b100485aa08) for lock 0x68a7113811d80588 00000080:00010000:2.0:1769074741.587602:0:36484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80588 from it ffffaa9141543cb8 00010000:00010000:2.0:1769074741.587606:0:36484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.587612:0:36484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590131:0:36484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.590162:0:36484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.590414:0:36484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590432:0:36484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590440:0:36484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.590448:0:36484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.590452:0:36484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.590459:0:36484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9141543c10 00010000:00010000:2.0:1769074741.590463:0:36484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590471:0:36484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590488:0:36484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074741.590499:0:36484:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=0 00000080:00200000:2.0:1769074741.590504:0:36484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074741.590511:0:36484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074741.590517:0:36484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590524:0:36484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590529:0:36484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074741.590533:0:36484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074741.590536:0:36484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80565 00000080:00010000:2.0:1769074741.590539:0:36484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80565 from it ffffaa9141543d68 00010000:00010000:2.0:1769074741.590542:0:36484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590549:0:36484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590557:0:36484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074741.590562:0:36484:0:(namei.c:2354:ll_unlink()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074741.590592:0:36484:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590617:0:36484:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074741.590628:0:36484:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x110:0x0]: losing layout lock 00010000:00010000:2.0:1769074741.590682:0:36484:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590690:0:36484:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590696:0:36484:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074741.590699:0:36484:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80588 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063e5c expref: -99 pid: 36480 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074741.590715:0:36484:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d80565 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063e16 expref: -99 pid: 36016 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074741.590733:0:36484:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074741.590745:0:36484:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5ad80 x1855007612416128/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074741.590767:0:36484:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:36484:1855007612416128:192.168.204.102@tcp:36:rm.0 00000100:00100000:2.0:1769074741.590859:0:36484:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074741.598625:0:36484:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:36484:1855007612416128:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074741.598719:0:36484:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x110:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074741.598731:0:36484:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x110:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074741.598740:0:36484:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x110:0x0](ffff8b100485aa08) 00010000:00010000:1.0:1769074741.792889:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074742.819613:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074743.840737:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074744.871101:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074745.888742:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074746.018513:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416384:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074746.018668:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416640:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074746.018706:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1769074746.018712:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612416256:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074746.018742:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416512:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074746.025466:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612416256:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074746.025488:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416512:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074746.025500:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416384:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074746.025510:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612416640:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074746.911973:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074747.935895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074748.960026:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074749.135676:0:37048:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 9: |X| create (same inum/gen) ====== 04:39:07 (1769074747) 00010000:00010000:1.0:1769074749.989563:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1769074750.640823:0:37068:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074750.640836:0:37068:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074750.640858:0:37068:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074750.640922:0:37068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37068 initiator: MDT0 00010000:00010000:1.0:1769074750.640932:0:37068:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37068 initiator: MDT0 00010000:00010000:1.0:1769074750.640941:0:37068:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37068 initiator: MDT0 00000100:00100000:1.0:1769074750.640968:0:37068:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612416768:192.168.204.102@tcp:101:lfs.0 00000100:00100000:1.0:1769074750.641063:0:37068:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074750.647181:0:37068:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 85899345922 gen 1 00000100:00100000:1.0:1769074750.647217:0:37068:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612416768:192.168.204.102@tcp:101:lfs.0 00010000:00010000:1.0:1769074750.647237:0:37068:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647250:0:37068:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647260:0:37068:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647270:0:37068:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647281:0:37068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647289:0:37068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647296:0:37068:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074750.647307:0:37068:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006137480 x1855007612416768/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074766 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074750.647326:0:37068:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647340:0:37068:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074750.647346:0:37068:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074750.647373:0:37068:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074750.647381:0:37068:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:1.0:1769074750.647387:0:37068:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074750.647390:0:37068:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9140d7fcb8 00010000:00010000:1.0:1769074750.647393:0:37068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.647401:0:37068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074750.648503:0:37068:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074750.648523:0:37068:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074750.648540:0:37068:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074750.648552:0:37068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.648564:0:37068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074750.648571:0:37068:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074750.648577:0:37068:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074750.648580:0:37068:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:1.0:1769074750.648585:0:37068:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9140d7fcb8 00010000:00010000:1.0:1769074750.648588:0:37068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.648595:0:37068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074750.649015:0:37068:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074750.649025:0:37068:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074750.649772:0:37068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.649787:0:37068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074750.649795:0:37068:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074750.649953:0:37068:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074750.649958:0:37068:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:1.0:1769074750.649965:0:37068:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9140d7fbb0 00010000:00010000:1.0:1769074750.649968:0:37068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074750.649976:0:37068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074750.649989:0:37068:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074750.649999:0:37068:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:1.0:1769074750.650275:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.650290:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00800000:00000004:1.0:1769074750.650298:0:37068:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1734, max_age 1742 00000100:00100000:1.0:1769074750.650345:0:37068:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612416896:192.168.204.102@tcp:41:lfs.0 00000100:00100000:1.0:1769074750.650431:0:37068:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074750.657682:0:37068:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612416896:192.168.204.102@tcp:41:lfs.0 00800000:00000004:1.0:1769074750.657723:0:37068:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:1.0:1769074750.657996:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658016:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658025:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658029:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658033:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658038:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658041:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658044:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658048:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658052:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658056:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658060:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658064:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658544:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658554:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658558:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658563:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658566:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658570:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658573:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658577:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658581:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658585:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658588:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658590:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658593:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658597:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658600:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658603:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658607:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658610:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658613:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658617:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658620:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658623:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658625:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658629:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658632:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658635:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658683:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658689:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658693:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658697:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658701:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658705:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658708:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658713:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658716:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658720:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658724:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658728:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658732:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658735:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658739:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658743:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658746:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658750:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658754:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658758:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658762:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658768:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658772:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658775:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658779:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658782:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658786:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658789:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658793:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.658796:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.658800:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659314:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659323:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659328:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659331:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659335:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659338:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659341:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659344:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659347:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659350:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659354:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659357:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659360:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659363:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659368:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659371:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659375:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659378:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659381:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659383:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659387:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659390:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659393:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659396:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659400:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659403:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659409:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659413:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659417:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659421:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659425:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659429:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659434:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659437:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659441:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659445:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659449:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659453:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659456:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659459:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659462:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659465:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659468:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659472:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659475:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659478:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659481:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659485:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659488:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659491:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659495:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659499:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659503:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659507:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659510:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659513:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659517:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00800000:00000080:1.0:1769074750.659520:0:37068:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00000080:00200080:1.0:1769074750.659523:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:1.0:1769074750.659530:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40400 uarg=0000000000000000 00020000:00000004:1.0:1769074750.659539:0:37068:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1734, max_age 1742 00000100:00100000:1.0:1769074750.659596:0:37068:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612417024:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074750.660241:0:37068:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a29c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074750.670634:0:37068:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612417024:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074750.670690:0:37068:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074750.670969:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.670991:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00020000:00000004:3.0:1769074750.671000:0:37068:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1734, max_age 1742 00000100:00100000:3.0:1769074750.671037:0:37068:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612417152:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074750.671278:0:37068:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729000 going to sleep for 11 seconds 00000100:00100000:3.0:1769074750.681291:0:37068:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37068:1855007612417152:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074750.681322:0:37068:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074750.682183:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682202:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682210:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682214:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682218:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682221:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682226:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682229:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682233:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682237:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682241:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682244:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682248:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682251:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682255:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682258:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682262:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682265:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682269:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682272:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682275:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682279:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682283:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682287:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682290:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682293:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682297:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682300:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682304:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682308:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682312:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682316:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682319:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682323:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682326:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682329:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682332:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682335:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682338:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682341:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682345:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682349:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682352:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682356:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682360:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682363:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682367:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682370:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682374:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682378:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682382:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682386:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682390:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682393:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682398:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682401:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682405:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682409:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682413:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682416:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682420:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682424:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682428:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682431:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682435:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682439:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682442:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682445:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682449:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682453:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682457:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682461:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682465:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682469:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682473:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682477:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682480:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682484:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682487:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682491:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682494:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682497:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682500:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682502:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682505:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682508:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682511:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682514:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682518:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682522:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682526:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682529:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682532:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682536:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682539:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682542:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682546:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682549:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682555:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682558:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682563:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682566:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682570:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682574:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682578:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682582:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682586:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682590:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682593:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682596:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682600:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682603:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682606:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682609:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682613:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682616:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682619:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682623:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682627:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682630:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682635:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682659:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682666:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682671:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074750.682675:0:37068:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc06aa2540 00020000:00000080:3.0:1769074750.682679:0:37068:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200008:3.0:1769074750.682708:0:37068:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074750.739358:0:37075:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074750.739387:0:37075:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074750.739404:0:37075:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739421:0:37075:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739429:0:37075:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074750.739438:0:37075:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074750.739442:0:37075:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:2.0:1769074750.739449:0:37075:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9141df3bd0 00010000:00010000:2.0:1769074750.739453:0:37075:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739462:0:37075:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739477:0:37075:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074750.739493:0:37075:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074750.739496:0:37075:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074750.739502:0:37075:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074750.739509:0:37075:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739517:0:37075:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739523:0:37075:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074750.739527:0:37075:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074750.739529:0:37075:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:2.0:1769074750.739533:0:37075:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9141df3cd0 00010000:00010000:2.0:1769074750.739535:0:37075:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739542:0:37075:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739550:0:37075:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074750.739556:0:37075:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074750.739562:0:37075:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074750.739567:0:37075:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739575:0:37075:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739581:0:37075:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074750.739585:0:37075:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074750.739587:0:37075:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8058f 00000080:00010000:2.0:1769074750.739590:0:37075:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8058f from it ffffaa9141df3db0 00010000:00010000:2.0:1769074750.739593:0:37075:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.739600:0:37075:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.739608:0:37075:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074750.739613:0:37075:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074750.739617:0:37075:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074750.739974:0:37075:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.740026:0:37075:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.740034:0:37075:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074750.740036:0:37075:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9c00/0x68a7113811d8058f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063e86 expref: -99 pid: 37068 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074750.740060:0:37075:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074750.740179:0:37075:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58e00 x1855007612417408/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074750.740203:0:37075:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37075:1855007612417408:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074750.740320:0:37075:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074750.755166:0:37075:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37075:1855007612417408:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074750.755220:0:37075:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x111:0x0](ffff8b100486db88) 00000080:00200000:2.0:1769074750.755265:0:37075:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:2.0:1769074750.776512:0:37078:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074750.776541:0:37078:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074750.776597:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37078 initiator: MDT0 00010000:00010000:2.0:1769074750.776605:0:37078:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37078 initiator: MDT0 00010000:00010000:2.0:1769074750.776613:0:37078:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37078 initiator: MDT0 00000100:00100000:2.0:1769074750.776632:0:37078:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417536:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074750.776756:0:37078:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442840 going to sleep for 11 seconds 00000100:00100000:2.0:1769074750.781423:0:37078:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417536:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074750.781449:0:37078:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781461:0:37078:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781473:0:37078:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781484:0:37078:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781501:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781520:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781527:0:37078:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074750.781537:0:37078:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59f80 x1855007612417536/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074766 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074750.781550:0:37078:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781562:0:37078:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074750.781569:0:37078:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074750.781596:0:37078:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074750.781603:0:37078:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80596 00000080:00010000:2.0:1769074750.781608:0:37078:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074750.781612:0:37078:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80596 from it ffffaa9141c33b58 00010000:00010000:2.0:1769074750.781615:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074750.781623:0:37078:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074750.781686:0:37078:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074750.781716:0:37078:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074750.781720:0:37078:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074750.781740:0:37078:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074750.781791:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37078 initiator: MDT0 00010000:00010000:2.0:1769074750.781798:0:37078:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37078 initiator: MDT0 00010000:00010000:2.0:1769074750.781816:0:37078:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37078 initiator: MDT0 00000100:00100000:2.0:1769074750.781836:0:37078:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417664:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074750.781896:0:37078:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074750.790003:0:37078:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417664:192.168.204.102@tcp:101:rm.0 00010000:00010000:0.0:1769074750.790029:0:37078:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790052:0:37078:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790077:0:37078:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790089:0:37078:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790100:0:37078:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790112:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790122:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790137:0:37078:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074750.790149:0:37078:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59f80 x1855007612417664/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074766 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074750.790162:0:37078:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790172:0:37078:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/273 (0 0) 00000080:00200000:0.0:1769074750.790179:0:37078:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074750.790201:0:37078:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x111:0x0], updating i_size 0 00000080:00200000:0.0:1769074750.790203:0:37078:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x111:0x0](ffff8b100486db88): rc = 0 00000080:00200000:0.0:1769074750.790218:0:37078:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x111:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074750.790224:0:37078:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x111:0x0](ffff8b100486db88) for lock 0x68a7113811d8059d 00000080:00010000:0.0:1769074750.790235:0:37078:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x111:0x0](ffff8b100486db88) 00000080:00010000:0.0:1769074750.790238:0:37078:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8059d from it ffffaa9141c33b20 00010000:00010000:0.0:1769074750.790242:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790250:0:37078:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074750.790290:0:37078:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x111:0x0](ffff8b100486db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074750.790296:0:37078:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x111:0x0](ffff8b100486db88),name=fsa-oleg402-client.virtnet 00000002:00010000:0.0:1769074750.790314:0:37078:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x111:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074750.790322:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790331:0:37078:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790338:0:37078:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074750.790342:0:37078:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x111:0x0](ffff8b100486db88) 00000080:00010000:0.0:1769074750.790346:0:37078:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x111:0x0](ffff8b100486db88) for lock 0x68a7113811d8059d 00000080:00010000:0.0:1769074750.790349:0:37078:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8059d from it ffffaa9141c33cb8 00010000:00010000:0.0:1769074750.790352:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790359:0:37078:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790423:0:37078:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074750.790430:0:37078:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074750.790437:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790445:0:37078:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790450:0:37078:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074750.790454:0:37078:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074750.790456:0:37078:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80596 00000080:00010000:0.0:1769074750.790459:0:37078:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80596 from it ffffaa9141c33c10 00010000:00010000:0.0:1769074750.790462:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790468:0:37078:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790476:0:37078:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074750.790483:0:37078:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:0.0:1769074750.790487:0:37078:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074750.790493:0:37078:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074750.790498:0:37078:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790505:0:37078:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790527:0:37078:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074750.790531:0:37078:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074750.790534:0:37078:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80596 00000080:00010000:0.0:1769074750.790537:0:37078:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80596 from it ffffaa9141c33d68 00010000:00010000:0.0:1769074750.790540:0:37078:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790547:0:37078:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790554:0:37078:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074750.790559:0:37078:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074750.790577:0:37078:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790598:0:37078:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074750.790608:0:37078:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x111:0x0]: losing layout lock 00010000:00010000:0.0:1769074750.790676:0:37078:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790684:0:37078:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790689:0:37078:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074750.790692:0:37078:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d8059d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3063ebe expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074750.790712:0:37078:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d80596 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063ea2 expref: -99 pid: 37078 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074750.790730:0:37078:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074750.790737:0:37078:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006024a80 x1855007612417920/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074750.790760:0:37078:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417920:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074750.790821:0:37078:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5cab40 going to sleep for 11 seconds 00000100:00100000:0.0:1769074750.802094:0:37078:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37078:1855007612417920:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074750.802203:0:37078:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x111:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074750.802217:0:37078:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x111:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074750.802227:0:37078:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x111:0x0](ffff8b100486db88) 00010000:00010000:1.0:1769074751.007791:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074751.136224:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612418048:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074751.138991:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074751.139017:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612418176:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074751.139979:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074751.140001:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612418304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074751.147298:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612418304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074751.147353:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612418176:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074751.157892:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612418048:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074752.039540:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074753.063302:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074754.058626:0:37109:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00010000:00010000:1.0:1769074754.082339:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074754.087719:0:37110:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074754.087774:0:37110:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074754.087875:0:37110:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37110 initiator: MDT0 00010000:00010000:2.0:1769074754.087890:0:37110:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37110 initiator: MDT0 00010000:00010000:2.0:1769074754.087902:0:37110:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37110 initiator: MDT0 00000100:00100000:2.0:1769074754.087939:0:37110:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37110:1855007612418432:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:2.0:1769074754.088082:0:37110:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10094429c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074754.105955:0:37110:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37110:1855007612418432:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:2.0:1769074754.106049:0:37110:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106150:0:37110:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106166:0:37110:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106179:0:37110:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106282:0:37110:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106292:0:37110:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106300:0:37110:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074754.106313:0:37110:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59f80 x1855007612418432/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074770 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074754.106329:0:37110:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074754.106345:0:37110:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074754.106352:0:37110:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074754.106474:0:37110:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:0.0:1769074754.106484:0:37110:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805a4 00000080:00010000:0.0:1769074754.106490:0:37110:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074754.106495:0:37110:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805a4 from it ffffaa9141cb3bd0 00010000:00010000:0.0:1769074754.106499:0:37110:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.106507:0:37110:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074754.106540:0:37110:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074754.106585:0:37110:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:0.0:1769074754.106590:0:37110:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074754.106608:0:37110:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074754.106617:0:37110:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.106628:0:37110:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074754.107171:0:37110:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074754.107176:0:37110:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074754.107180:0:37110:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805a4 00000080:00010000:0.0:1769074754.107185:0:37110:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805a4 from it ffffaa9141cb3cd0 00010000:00010000:0.0:1769074754.107274:0:37110:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.107284:0:37110:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074754.107295:0:37110:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074754.107305:0:37110:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074754.107317:0:37110:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074754.107324:0:37110:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.107332:0:37110:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074754.107338:0:37110:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074754.107342:0:37110:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074754.107344:0:37110:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805a4 00000080:00010000:0.0:1769074754.107348:0:37110:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805a4 from it ffffaa9141cb3db0 00010000:00010000:0.0:1769074754.107350:0:37110:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.107358:0:37110:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074754.107365:0:37110:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074754.107371:0:37110:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:0.0:1769074754.107381:0:37110:0:(namei.c:2020:ll_mknod()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:0.0:1769074754.107411:0:37110:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.107472:0:37110:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074754.107479:0:37110:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074754.107483:0:37110:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d805a4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3063ee8 expref: -99 pid: 37110 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074754.107516:0:37110:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074754.107529:0:37110:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006024000 x1855007612418688/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074754.107560:0:37110:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37110:1855007612418688:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:0.0:1769074754.108085:0:37110:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5cab40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074754.125169:0:37110:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:37110:1855007612418688:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074754.125348:0:37110:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00200000:2.0:1769074754.125417:0:37110:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f9.replay-single, unhashed 1 00000080:00200000:1.0:1769074754.153395:0:37112:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074754.153434:0:37112:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074754.153527:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37112 initiator: MDT0 00010000:00010000:1.0:1769074754.153541:0:37112:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37112 initiator: MDT0 00010000:00010000:1.0:1769074754.153553:0:37112:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37112 initiator: MDT0 00000100:00100000:1.0:1769074754.153591:0:37112:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37112:1855007612418816:192.168.204.102@tcp:101:ls.0 00000100:00100000:1.0:1769074754.153757:0:37112:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2d80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074754.160171:0:37112:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37112:1855007612418816:192.168.204.102@tcp:101:ls.0 00010000:00010000:3.0:1769074754.160315:0:37112:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160338:0:37112:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160354:0:37112:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160371:0:37112:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160385:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160398:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160406:0:37112:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074754.160418:0:37112:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134700 x1855007612418816/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074770 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074754.160434:0:37112:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160450:0:37112:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074754.160457:0:37112:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074754.160503:0:37112:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074754.160510:0:37112:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ab 00000080:00010000:3.0:1769074754.160517:0:37112:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074754.160521:0:37112:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ab from it ffffaa9141c6bb58 00010000:00010000:3.0:1769074754.160529:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.160538:0:37112:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.160576:0:37112:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074754.160609:0:37112:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=9 00000080:00200000:3.0:1769074754.160614:0:37112:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074754.161313:0:37112:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074754.161391:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37112 initiator: MDT0 00010000:00010000:3.0:1769074754.161400:0:37112:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37112 initiator: MDT0 00010000:00010000:3.0:1769074754.161412:0:37112:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37112 initiator: MDT0 00000100:00100000:3.0:1769074754.161445:0:37112:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37112:1855007612418944:192.168.204.102@tcp:101:ls.0 00000100:00100000:3.0:1769074754.161565:0:37112:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729000 going to sleep for 11 seconds 00000100:00100000:3.0:1769074754.167164:0:37112:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37112:1855007612418944:192.168.204.102@tcp:101:ls.0 00010000:00010000:3.0:1769074754.167209:0:37112:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167239:0:37112:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167248:0:37112:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167263:0:37112:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167276:0:37112:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167290:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167298:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167305:0:37112:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074754.167317:0:37112:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037ddf80 x1855007612418944/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074770 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074754.167332:0:37112:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167348:0:37112:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/274 (0 0) 00000080:00200000:3.0:1769074754.167357:0:37112:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074754.167390:0:37112:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x112:0x0], updating i_size 0 00000080:00200000:3.0:1769074754.167393:0:37112:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x112:0x0](ffff8b100486d348): rc = 0 00000080:00200000:3.0:1769074754.167414:0:37112:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x112:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074754.167420:0:37112:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805b2 00000080:00010000:3.0:1769074754.167435:0:37112:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074754.167439:0:37112:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805b2 from it ffffaa9141c6bb20 00010000:00010000:3.0:1769074754.167442:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167451:0:37112:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074754.167499:0:37112:0:(file.c:6252:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffff8b100486d348), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1769074754.167508:0:37112:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348),name=f9.replay-single 00000002:00010000:3.0:1769074754.167534:0:37112:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074754.167544:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167554:0:37112:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.167560:0:37112:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074754.167565:0:37112:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074754.167567:0:37112:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805b2 00000080:00010000:3.0:1769074754.167571:0:37112:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805b2 from it ffffaa9141c6bcb8 00010000:00010000:3.0:1769074754.167574:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167580:0:37112:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.167628:0:37112:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074754.167636:0:37112:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074754.167680:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167688:0:37112:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.167695:0:37112:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074754.167699:0:37112:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074754.167702:0:37112:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ab 00000080:00010000:3.0:1769074754.167705:0:37112:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ab from it ffffaa9141c6bb58 00010000:00010000:3.0:1769074754.167708:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167715:0:37112:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.167722:0:37112:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074754.167730:0:37112:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:3.0:1769074754.167736:0:37112:0:(file.c:6252:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffff8b100486d348), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1769074754.167740:0:37112:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348),name=f9.replay-single 00000002:00010000:3.0:1769074754.167748:0:37112:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074754.167754:0:37112:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167762:0:37112:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074754.167768:0:37112:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074754.167771:0:37112:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074754.167774:0:37112:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805b2 00000080:00010000:3.0:1769074754.167777:0:37112:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805b2 from it ffffaa9141c6bcb8 00010000:00010000:3.0:1769074754.167780:0:37112:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074754.167786:0:37112:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074755.104588:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074756.130986:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074756.256296:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612419072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.256439:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419328:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.256480:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074756.256487:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.256515:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.267346:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419456:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.267380:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612419072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074756.267392:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419328:192.168.204.102@tcp:400:kworker.0 00000100:00080000:3.0:1769074756.277081:0:2366:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffff8b101127b800 of lustre-MDT0000_UUID@192.168.204.102@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1769074756.277092:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1769074756.277100:0:2366:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1769074756.277108:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074756.277113:0:2366:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1723 00000100:00080000:3.0:1769074756.277127:0:2366:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074756.277140:0:2366:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074756.277176:0:2366:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612419584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074756.277212:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612419200:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074756.277265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612419584:192.168.204.102@tcp:38:ptlrpcd_01_00.0 00000100:00080000:2.0:1769074756.280193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074756.280213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-19) 00000100:00100000:2.0:1769074756.280220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612419584:192.168.204.102@tcp:38:ptlrpcd_01_00.0 00010000:00010000:3.0:1769074757.162957:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074758.175880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074759.199916:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074760.224447:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074761.254116:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769074761.376998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074761.377007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074761.377012:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074761.377025:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1749 00000100:00080000:0.0:1769074761.377040:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1769074761.377047:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074761.377063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074761.377085:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612419840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074761.377156:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612419968:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074761.377272:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074761.377278:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612419712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074761.377316:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612420096:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074761.377634:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612419840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074761.382457:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612420096:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074761.382488:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612419968:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074762.272443:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074763.298721:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074764.321668:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074765.346526:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074766.367907:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074766.497017:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074766.497029:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769074766.499388:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612420224:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074766.499565:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612420480:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074766.499606:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074766.499615:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612420352:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074766.534720:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612420352:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074766.534747:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612420480:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074767.393969:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074768.419074:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074769.442133:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074770.466493:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074771.488102:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074771.616094:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074771.616104:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769074771.616120:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:1.0:1769074771.616126:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b101127b800: disconnect after 21s idle 00000100:00080000:1.0:1769074771.616140:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:1.0:1769074771.616143:0:38:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b101127b800: disconnect after 21s idle 00000100:00100000:0.0:1769074771.616308:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612420736:192.168.204.102@tcp:9:kworker.0 00000100:00100000:0.0:1769074771.616458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612420864:192.168.204.102@tcp:9:kworker.0 00000100:00100000:0.0:1769074771.616620:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612420608:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074771.637954:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006134a80 x1855007612420736/t0(0) o9->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074781 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074771.637979:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1769074771.637984:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612420736:192.168.204.102@tcp:9:kworker.0 00000100:00080000:2.0:1769074771.638009:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006137100 x1855007612420864/t0(0) o9->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074781 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074771.638025:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1769074771.638029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612420864:192.168.204.102@tcp:9:kworker.0 00010000:00010000:2.0:1769074772.511950:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074773.539350:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074774.560208:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074775.587581:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074776.609097:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074776.735905:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074761/real 1769074761] req@ffff8b1006025180 x1855007612419840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074776 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074776.735954:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612419840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074776 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074776.735972:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074776.735984:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074776.736012:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612419840:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074776.737905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074776.737913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074776.737918:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074776.737929:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1754 00000100:00080000:3.0:1769074776.737943:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1769074776.737952:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074776.737965:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074776.737984:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612421120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074776.738098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612421120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769074776.738195:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612420992:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074776.741975:0:2367:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.102@tcp of MGS@192.168.204.102@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1769074776.741985:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1769074776.741992:0:2367:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1769074776.741996:0:2367:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1769074776.741999:0:2367:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1769074776.742009:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006025880 x1855007612419712/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074777 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074776.742038:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d59c00 x1855007612420224/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074782 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074776.742065:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006136a00 x1855007612420608/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074787 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074776.742073:0:2367:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037dce00 x1855007612420992/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074792 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1769074776.742083:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1769074776.742086:0:2367:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1769074776.742088:0:2367:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1769074776.742092:0:2367:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074776.742096:0:2367:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1723 00000100:00080000:2.0:1769074776.742108:0:2367:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769074776.742155:0:2367:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612421248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074776.742196:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612420992:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074776.742279:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612420608:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074776.742290:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612419712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074776.742687:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612420224:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074776.746766:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612421248:192.168.204.102@tcp:250:ptlrpcd_01_01.0 00000100:00080000:0.0:1769074776.751775:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:0.0:1769074776.751800:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:0.0:1769074776.751812:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3063c2c to 0x143201ebf3063f43 00000100:00080000:0.0:1769074776.751818:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:0.0:1769074776.751826:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:2.0:1769074776.752592:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612421248:192.168.204.102@tcp:250:ptlrpcd_01_01.0 00000100:00080000:0.0:1769074776.752796:0:37302:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:0.0:1769074776.752801:0:37302:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:0.0:1769074776.752804:0:37302:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:0.0:1769074776.752824:0:37302:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074776.752837:0:37302:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074776.752844:0:37302:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074776.752921:0:37302:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074776.752934:0:37302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169e00/0x68a7113811d80573 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063e32 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074776.752961:0:37302:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074776.752968:0:37302:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074776.752980:0:37302:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074776.754512:0:37302:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074776.754528:0:37302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029aca00/0x68a7113811d80581 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063e40 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074776.754560:0:37302:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074776.754568:0:37302:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074776.754575:0:37302:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074776.754908:0:37302:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074776.754914:0:37302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d8056c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063e2b expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074776.754930:0:37302:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074776.754936:0:37302:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074776.754942:0:37302:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:0.0:1769074776.755315:0:37302:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074776.755321:0:37302:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d8057a lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063e39 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:0.0:1769074776.755342:0:37302:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:0.0:1769074776.755350:0:37302:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:0.0:1769074776.755354:0:37302:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:0.0:1769074776.755430:0:37302:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:0.0:1769074776.755441:0:37302:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:3.0:1769074777.631947:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:1.0:1769074778.207820:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:1.0:1769074778.207831:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 26 10000000:01000000:1.0:1769074778.207839:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:1.0:1769074778.207905:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.207919:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.208509:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074778.208573:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421376:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1769074778.208764:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.212336:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421376:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1769074778.212382:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.212397:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074778.212500:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421504:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.212618:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.260236:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421504:192.168.204.102@tcp:256:ll_cfg_requeue.0 10000000:01000000:0.0:1769074778.260309:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:0.0:1769074778.260322:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.260335:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074778.260347:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1769074778.260350:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:0.0:1769074778.260354:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1769074778.260404:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.260414:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.260425:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074778.260452:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421632:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.260554:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.264148:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421632:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1769074778.264203:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.264222:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074778.264368:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421760:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.264534:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.270109:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421760:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.270212:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421888:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.270922:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.273880:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612421888:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1769074778.276895:0:37303:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:1.0:1769074778.278480:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:1.0:1769074778.278523:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1769074778.278539:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.278567:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:1.0:1769074778.278580:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:1.0:1769074778.278583:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:1.0:1769074778.278587:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:1.0:1769074778.278715:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.278724:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.278737:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074778.278777:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422016:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1769074778.278942:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2a80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.287982:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422016:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1769074778.288019:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.288033:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074778.288154:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422144:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.288247:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074778.293114:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422144:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074778.293222:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422272:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074778.293351:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442840 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.296988:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422272:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1769074778.298147:0:37304:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:1.0:1769074778.298626:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:1.0:1769074778.298692:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:1.0:1769074778.298707:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.298726:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:1.0:1769074778.298739:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:1.0:1769074778.298743:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:1.0:1769074778.298746:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:1.0:1769074778.298796:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.298805:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.298816:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074778.298849:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422400:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1769074778.298959:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2240 going to sleep for 11 seconds 00000100:00100000:0.0:1769074778.305886:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422400:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:0.0:1769074778.305926:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074778.305941:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074778.306046:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422528:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:0.0:1769074778.306162:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100a5ca000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074778.310286:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612422528:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1769074778.310360:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1769074778.310375:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074778.310390:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074778.655884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074779.681005:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074780.705692:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074781.728165:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074781.856222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074781.856233:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074781.856319:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422656:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074781.861861:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422656:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074782.752249:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074783.776148:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074784.801148:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074785.060547:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:0.0:1769074785.061060:0:30958:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624740352 00000100:00100000:0.0:1769074785.061076:0:30958:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624740352 00000100:00100000:0.0:1769074785.061153:0:30958:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 199 00000100:00100000:0.0:1769074785.061165:0:30958:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006135f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:86724556-597d-4d0e-ac75-6d9d64145050+3:40564:x1855007624740352:12345-192.168.204.102@tcp:104: 00010000:00010000:0.0:1769074785.061188:0:30958:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074785.061297:0:30958:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074785.061305:0:30958:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006135f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:86724556-597d-4d0e-ac75-6d9d64145050+3:40564:x1855007624740352:12345-192.168.204.102@tcp:104: Request processed in 142us (775us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074785.061314:0:30958:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 199 00010000:00010000:3.0:1769074785.061489:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074785.061498:0:29100:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074785.061508:0:29100:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b9e00 already unused, calling callback (ffffffffc121fc20) 10000000:00010000:3.0:1769074785.061513:0:29100:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074785.061523:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074785.061532:0:29100:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074785.061538:0:29100:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074785.061609:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074785.061616:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074785.061672:0:29100:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074785.061679:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10060b9e00/0x68a7113811d805b9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf3063f4a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074785.061700:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074785.061702:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074785.061704:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074785.061945:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074785.061977:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422784:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074785.064860:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422784:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074785.825820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074786.852770:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074786.976868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074786.976878:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769074786.976917:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422912:192.168.204.102@tcp:400:kworker.0 10000000:01000000:3.0:1769074786.979296:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1769074786.979309:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 26 10000000:01000000:3.0:1769074786.979318:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1769074786.979380:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074786.979395:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074786.979408:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074786.979450:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612423040:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074786.980837:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074786.981474:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612422912:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074786.987500:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612423040:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074786.988415:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074786.988433:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074786.989047:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612423168:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:0.0:1769074786.990262:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729180 going to sleep for 11 seconds 00000100:00100000:0.0:1769074787.027574:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612423168:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:0.0:1769074787.027707:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:0.0:1769074787.027958:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:0.0:1769074787.027973:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074787.027987:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:0.0:1769074787.080356:0:37353:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 60, was 0): CONNECTING 00010000:00080000:0.0:1769074787.080394:0:37353:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8b1008679800@lustre-MDT0000-mdc-ffff8b101127b800: found existing conn 192.168.204.102@tcp, moved to head 00010000:00010000:2.0:1769074787.872706:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074788.898075:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074789.921220:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074790.944616:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074791.968729:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074792.096787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074792.096797:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769074792.096818:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074792.096843:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612423296:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074792.123949:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612423296:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074792.992481:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074794.017589:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074794.426061:0:37405:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074794.426075:0:37405:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074794.426094:0:37405:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074794.426111:0:37405:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.426127:0:37405:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074794.426134:0:37405:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074794.426142:0:37405:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074794.426146:0:37405:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ab 00000080:00010000:0.0:1769074794.426152:0:37405:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ab from it ffffaa9141f8bcb8 00010000:00010000:0.0:1769074794.426157:0:37405:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.426166:0:37405:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074794.427130:0:37405:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074794.427139:0:37405:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074794.427153:0:37405:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074794.427165:0:37405:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.427178:0:37405:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074794.427185:0:37405:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074794.427192:0:37405:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074794.427195:0:37405:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ab 00000080:00010000:0.0:1769074794.427200:0:37405:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ab from it ffffaa9141f8bcb8 00010000:00010000:0.0:1769074794.427204:0:37405:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.427212:0:37405:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074794.427440:0:37405:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074794.427448:0:37405:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074794.427455:0:37405:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.427464:0:37405:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074794.427472:0:37405:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074794.427476:0:37405:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074794.427479:0:37405:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ab 00000080:00010000:0.0:1769074794.427483:0:37405:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ab from it ffffaa9141f8bbb0 00010000:00010000:0.0:1769074794.427486:0:37405:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074794.427494:0:37405:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074794.427503:0:37405:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:0.0:1769074794.427510:0:37405:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:0.0:1769074794.427799:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:0.0:1769074794.427816:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1006029400 uarg=0000000000000000 00800000:00000004:0.0:1769074794.427823:0:37405:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1743, max_age 1786 00000100:00080000:0.0:1769074794.427869:0:37405:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1006027100 x1855007612423424/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074794.427881:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c566c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769074795.042535:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074795.488477:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c566c0 going to sleep for 0 seconds 00010000:00010000:2.0:1769074796.066144:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074796.193138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074776/real 1769074776] req@ffff8b10037dea00 x1855007612421120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074796 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074796.193171:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612421120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074796 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.193192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769074796.193224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769074796.193229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612421120:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074796.193413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074796.193418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074796.193421:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074796.193427:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 0 00000100:00080000:3.0:1769074796.193440:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074796.193450:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074796.193465:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612423680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074796.193565:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612423552:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074796.194075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769074796.198712:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612423552:192.168.204.102@tcp:400:kworker.0 00010000:00080000:0.0:1769074796.208187:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b10037dce00 x1855007612423680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074806 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074796.208204:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 85899345924 gen 1 00000100:00100000:0.0:1769074796.208232:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006024000 x1855007612418688/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074770 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.208246:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 21 00000100:00080000:0.0:1769074796.208262:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:0.0:1769074796.208267:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:0.0:1769074796.208269:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 85899345924 gen 1 00000100:00100000:0.0:1769074796.208272:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006024000 x1855007612418688/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074770 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.208279:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 85899345924 last 0 00000100:00080000:0.0:1769074796.208285:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1006024000 x1855007612418688/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074770 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074796.208299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769074796.208335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612418688:192.168.204.102@tcp:36:mcreate.0 00000100:00080000:0.0:1769074796.266821:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1006024000 x1855007612418688/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074812 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.266832:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:0.0:1769074796.266834:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 85899345924 gen 1 00000100:00100000:0.0:1769074796.266838:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006024000 x1855007612418688/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074812 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.266843:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 85899345924 last 85899345925 00000100:00080000:0.0:1769074796.266847:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:0.0:1769074796.267405:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612418688:192.168.204.102@tcp:36:mcreate.0 00010000:00080000:2.0:1769074796.276192:0:37420:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:2.0:1769074796.276200:0:37420:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (2) 00010000:00010000:2.0:1769074796.276243:0:37420:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d805ab lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3063f04 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074796.276281:0:37420:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x112:0x0]: losing layout lock 00010000:00010000:2.0:1769074796.276307:0:37420:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168800/0x68a7113811d805b2 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3063f20 expref: -99 pid: 37112 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074796.276320:0:37420:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:2.0:1769074796.276837:0:37420:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:0.0:1769074796.277194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423808:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:0.0:1769074796.393309:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 85899345925 gen 1 00000100:00080000:0.0:1769074796.393337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:0.0:1769074796.393349:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1005d5b480 x1855007612423808/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074812 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.393358:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1005d5b480 x1855007612423808/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074812 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074796.393375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1769074796.396717:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:1.0:1769074796.396733:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 0000000033be2800): req@ffff8b1006027100 x1855007612423424/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074796.396767:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423808:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:2.0:1769074796.396989:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c566c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074796.401478:0:37405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37405:1855007612423424:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074796.401508:0:37405:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:2.0:1769074796.401594:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401611:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401619:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401623:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401627:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401631:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401634:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401682:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401689:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401693:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401697:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401701:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401705:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401708:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401712:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401716:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401720:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401724:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401727:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401731:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401735:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401739:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401742:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401746:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401749:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401752:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401756:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401759:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401762:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401766:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401770:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401773:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401777:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401780:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401783:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401786:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401790:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401793:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401797:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401800:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401804:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401807:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401811:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401814:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401818:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401821:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401825:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401828:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401832:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401835:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401839:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401842:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401846:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401850:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401853:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401857:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401860:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401863:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401868:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401871:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401876:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401880:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401884:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401887:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401891:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401895:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401899:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401902:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401906:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401910:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401913:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401917:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401920:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401923:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401927:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401930:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401935:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401938:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401941:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401944:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401948:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401951:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401955:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401958:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401962:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401966:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401969:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401973:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401977:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401979:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.401984:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.401987:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403273:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403289:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403294:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403297:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403301:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403304:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403308:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403311:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403315:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403319:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403322:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403326:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403329:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403333:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403337:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403340:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.403344:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.403348:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404539:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404552:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404557:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404561:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404564:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404568:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404571:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404574:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404578:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404581:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404584:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404588:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404591:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404595:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404598:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404602:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.404605:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00800000:00000080:2.0:1769074796.404608:0:37405:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00000080:00200080:2.0:1769074796.405300:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:2.0:1769074796.405317:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8400 uarg=0000000000000000 00020000:00000004:2.0:1769074796.405325:0:37405:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1743, max_age 1788 00000100:00080000:2.0:1769074796.405342:0:37405:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1725 00000100:00080000:2.0:1769074796.405357:0:37405:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074796.406308:0:37405:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:2.0:1769074796.406341:0:37405:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58700 x1855007612423936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074796.406405:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423936:192.168.204.102@tcp:8:lfs.0 00000100:00080000:2.0:1769074796.406521:0:37405:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1005d5b100 x1855007612424064/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074796.406531:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442540 going to sleep for 0 seconds 00000100:00100000:2.0:1769074796.415902:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934747 gen 9 00000100:00080000:2.0:1769074796.415919:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b101127b800: connect to target with instance 2 00000100:00080000:2.0:1769074796.415936:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1769074796.415947:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074796.415953:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074796.415957:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:2.0:1769074796.415960:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1769074796.415966:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074796.415968:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074796.415979:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000d28ebe46): req@ffff8b1005d5b100 x1855007612424064/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074796.416305:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074796.416896:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612423936:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074796.423713:0:37405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37405:1855007612424064:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074796.423744:0:37405:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074796.423816:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.423833:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00020000:00000004:3.0:1769074796.423841:0:37405:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1743, max_age 1788 00000100:00080000:3.0:1769074796.423853:0:37405:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1725 00000100:00080000:3.0:1769074796.423864:0:37405:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074796.423874:0:37405:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:3.0:1769074796.423899:0:37405:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612424192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074796.423951:0:37405:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b10037ddf80 x1855007612424320/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074796.423961:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729840 going to sleep for 0 seconds 00000100:00100000:2.0:1769074796.425289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612424192:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074796.433095:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967450 gen 9 00000100:00080000:2.0:1769074796.433111:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b101127b800: connect to target with instance 1 00000100:00080000:2.0:1769074796.433121:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1769074796.433132:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074796.433138:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074796.433144:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:2.0:1769074796.433148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1769074796.433155:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074796.433157:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074796.433170:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000adca437c): req@ffff8b10037ddf80 x1855007612424320/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074796.433199:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612424192:192.168.204.102@tcp:8:lfs.0 00000100:00100000:3.0:1769074796.433480:0:37405:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009729840 going to sleep for 11 seconds 00000100:00100000:3.0:1769074796.454770:0:37405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37405:1855007612424320:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074796.454800:0:37405:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074796.454879:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454897:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454905:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454909:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454913:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454917:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454921:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454925:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454929:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454932:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454937:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454941:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454945:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454949:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454953:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454956:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454960:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454963:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454967:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454970:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454974:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454977:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454981:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454985:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454989:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.454993:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.454997:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455000:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455004:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455008:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455012:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455016:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455020:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455023:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455027:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455031:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455036:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455040:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455044:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455048:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455052:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455055:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455059:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455062:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455065:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455069:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455073:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455077:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455080:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455083:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455087:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455090:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455094:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455097:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455101:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455105:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455109:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455113:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455116:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455120:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455124:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455128:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455132:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455135:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455139:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455143:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455147:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455151:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455155:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455158:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455162:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455166:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455170:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455174:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455178:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455181:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455186:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455190:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455194:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455198:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455202:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455206:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455210:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455214:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455218:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455223:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455227:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455231:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455235:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455239:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455242:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455247:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455250:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455254:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455258:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455263:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455267:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455270:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455273:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455276:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455280:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455300:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455305:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455307:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455311:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455314:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455317:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455321:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455325:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455329:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455334:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455337:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455341:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455344:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455349:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455353:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455357:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455360:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455392:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455397:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455402:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455406:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455409:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455412:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200080:3.0:1769074796.455416:0:37405:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc5ecfcb10 00020000:00000080:3.0:1769074796.455419:0:37405:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35800 uarg=0000000000000000 00000080:00200008:3.0:1769074796.455432:0:37405:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:3.0:1769074797.087807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074798.112680:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074799.136240:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074800.165373:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074800.870507:0:37658:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:3.0:1769074801.191980:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074801.313154:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612424576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074801.313238:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612424832:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.313271:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612424448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.313327:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612424704:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.320396:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612424448:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.320443:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612424576:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.320456:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612424832:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074801.337546:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612424704:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074802.207902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1769074802.674585:0:37812:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:2.0:1769074803.231886:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074804.256091:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074804.652577:0:37850:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074804.652590:0:37850:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074804.652613:0:37850:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074804.652717:0:37850:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37850 initiator: MDT0 00010000:00010000:0.0:1769074804.652729:0:37850:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37850 initiator: MDT0 00010000:00010000:0.0:1769074804.652739:0:37850:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37850 initiator: MDT0 00000100:00100000:0.0:1769074804.652764:0:37850:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612424960:192.168.204.102@tcp:101:lfs.0 00000100:00100000:0.0:1769074804.654410:0:37850:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:3.0:1769074804.659079:0:37850:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612424960:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074804.659104:0:37850:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659115:0:37850:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659134:0:37850:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659145:0:37850:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659156:0:37850:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659165:0:37850:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659172:0:37850:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074804.659182:0:37850:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024000 x1855007612424960/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074820 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074804.659195:0:37850:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659207:0:37850:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074804.659213:0:37850:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074804.659249:0:37850:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074804.659256:0:37850:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:3.0:1769074804.659262:0:37850:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074804.659265:0:37850:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa91414abcb8 00010000:00010000:3.0:1769074804.659268:0:37850:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.659277:0:37850:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074804.660371:0:37850:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074804.660378:0:37850:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074804.660395:0:37850:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.660407:0:37850:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.660418:0:37850:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.660425:0:37850:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.660430:0:37850:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074804.660434:0:37850:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:3.0:1769074804.660439:0:37850:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa91414abcb8 00010000:00010000:3.0:1769074804.660443:0:37850:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.660450:0:37850:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.660559:0:37850:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074804.660568:0:37850:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.660574:0:37850:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.660583:0:37850:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.660590:0:37850:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.660594:0:37850:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074804.660598:0:37850:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:3.0:1769074804.660602:0:37850:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa91414abbb0 00010000:00010000:3.0:1769074804.660605:0:37850:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.660612:0:37850:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.660621:0:37850:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074804.660629:0:37850:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074804.661571:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.661589:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00800000:00000004:2.0:1769074804.661595:0:37850:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1789, max_age 1796 00000100:00100000:2.0:1769074804.661635:0:37850:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425088:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074804.661745:0:37850:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074804.668224:0:37850:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425088:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074804.668261:0:37850:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:2.0:1769074804.668410:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668428:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668435:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668439:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668444:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668448:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668452:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668456:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668460:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668464:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668468:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668472:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668475:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668479:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668483:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668487:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668491:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668495:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668499:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668502:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668506:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668509:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668513:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668516:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668520:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668524:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668528:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668531:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668535:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668538:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668542:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668545:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668549:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668552:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668556:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668559:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668563:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668566:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668570:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668573:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668578:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668582:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668586:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668589:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668593:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668596:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668600:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668603:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668607:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668611:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668614:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668618:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668622:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668625:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668629:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668633:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668684:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668690:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668696:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668700:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668704:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668707:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668711:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668714:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668717:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668720:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668724:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668726:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668730:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668733:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668737:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668739:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668742:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668745:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668748:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668751:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668755:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668758:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668761:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668764:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668767:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668770:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668774:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668777:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668780:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668795:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668801:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668806:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668810:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668814:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668818:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668821:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668825:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668828:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668832:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668836:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668839:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668846:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668849:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668863:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668868:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668871:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668875:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668878:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668881:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668884:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668888:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668891:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668895:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668898:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668902:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668906:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668909:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668912:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668916:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668920:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668923:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668926:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668929:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668932:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668936:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668939:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668943:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668946:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668949:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668952:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668955:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00800000:00000080:2.0:1769074804.668959:0:37850:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.668962:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.668968:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00020000:00000004:2.0:1769074804.668976:0:37850:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1789, max_age 1796 00000100:00100000:2.0:1769074804.669025:0:37850:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425216:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074804.669106:0:37850:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074804.673389:0:37850:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425216:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074804.673421:0:37850:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074804.677140:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.677168:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00020000:00000004:2.0:1769074804.677178:0:37850:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1789, max_age 1796 00000100:00100000:2.0:1769074804.677237:0:37850:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425344:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074804.677672:0:37850:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074804.682321:0:37850:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:37850:1855007612425344:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074804.682350:0:37850:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074804.682474:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682491:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682499:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682503:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682507:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682510:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682514:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682518:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682522:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682526:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682530:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682533:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682537:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682541:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682545:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682549:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682552:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682555:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682558:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682562:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682566:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682570:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682573:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682576:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682579:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682582:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682586:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682589:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682593:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682597:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682600:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682604:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682608:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682611:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682615:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682618:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682622:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682625:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682629:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682631:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682635:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682671:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682677:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682680:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682684:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682687:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682691:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682695:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682699:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682703:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682706:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682710:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682714:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682717:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682721:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682725:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682728:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682732:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682735:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682739:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682743:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682746:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682750:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682753:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682758:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682761:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682765:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682769:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682773:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682777:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682780:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682785:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682801:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682805:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682810:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682814:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682818:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682821:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682825:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682828:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682832:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682836:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682839:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682842:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682846:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682849:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682864:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682868:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682871:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682874:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682878:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682882:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682885:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682889:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682893:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682896:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682900:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682903:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682907:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682910:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682913:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682916:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682920:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682924:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682927:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682931:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682935:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682939:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682944:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682947:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682951:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682954:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682958:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682961:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682965:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682968:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682972:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682975:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682978:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682981:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682985:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682989:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.682993:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.682996:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200080:2.0:1769074804.683000:0:37850:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc9c53c230 00020000:00000080:2.0:1769074804.683003:0:37850:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096ab000 uarg=0000000000000000 00000080:00200008:2.0:1769074804.683017:0:37850:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:3.0:1769074804.722550:0:37867:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074804.722575:0:37867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.722593:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.722608:0:37867:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.722617:0:37867:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.722624:0:37867:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074804.722628:0:37867:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:3.0:1769074804.722634:0:37867:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa914176bb58 00010000:00010000:3.0:1769074804.722664:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.722673:0:37867:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.722692:0:37867:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074804.722706:0:37867:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=9 00000080:00200000:3.0:1769074804.722711:0:37867:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074804.722725:0:37867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f9.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.722772:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37867 initiator: MDT0 00010000:00010000:3.0:1769074804.722780:0:37867:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37867 initiator: MDT0 00010000:00010000:3.0:1769074804.722789:0:37867:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37867 initiator: MDT0 00000100:00100000:3.0:1769074804.722815:0:37867:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37867:1855007612425472:192.168.204.102@tcp:101:ls.0 00000100:00100000:3.0:1769074804.722900:0:37867:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009ac9840 going to sleep for 11 seconds 00000100:00100000:3.0:1769074804.734865:0:37867:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ls:93598d0a-9484-4e40-94b2-393ee542cbad:37867:1855007612425472:192.168.204.102@tcp:101:ls.0 00010000:00010000:3.0:1769074804.734892:0:37867:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734917:0:37867:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734926:0:37867:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734937:0:37867:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734953:0:37867:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734965:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734972:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.734980:0:37867:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074804.734991:0:37867:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037ddf80 x1855007612425472/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074820 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074804.735003:0:37867:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.735012:0:37867:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/274 (0 0) 00000080:00200000:3.0:1769074804.735019:0:37867:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074804.735039:0:37867:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x112:0x0], updating i_size 0 00000080:00200000:3.0:1769074804.735042:0:37867:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x112:0x0](ffff8b100486d348): rc = 0 00000080:00200000:3.0:1769074804.735058:0:37867:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x112:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074804.735064:0:37867:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805e3 00000080:00010000:3.0:1769074804.735075:0:37867:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074804.735079:0:37867:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805e3 from it ffffaa914176bb20 00010000:00010000:3.0:1769074804.735082:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.735874:0:37867:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074804.735916:0:37867:0:(file.c:6252:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffff8b100486d348), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1769074804.735923:0:37867:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348),name=f9.replay-single 00000002:00010000:3.0:1769074804.735945:0:37867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.735955:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.735967:0:37867:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.735975:0:37867:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.735980:0:37867:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074804.735984:0:37867:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805e3 00000080:00010000:3.0:1769074804.735989:0:37867:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805e3 from it ffffaa914176bcb8 00010000:00010000:3.0:1769074804.735992:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.736001:0:37867:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.736042:0:37867:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074804.736050:0:37867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.736057:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.736071:0:37867:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.736077:0:37867:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.736081:0:37867:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074804.736084:0:37867:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:3.0:1769074804.736280:0:37867:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa914176bb58 00010000:00010000:3.0:1769074804.736286:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.736294:0:37867:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.736304:0:37867:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074804.736311:0:37867:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:3.0:1769074804.736317:0:37867:0:(file.c:6252:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffff8b100486d348), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1769074804.736322:0:37867:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348),name=f9.replay-single 00000002:00010000:3.0:1769074804.736331:0:37867:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074804.736337:0:37867:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.736346:0:37867:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074804.736353:0:37867:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074804.736357:0:37867:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:3.0:1769074804.736360:0:37867:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805e3 00000080:00010000:3.0:1769074804.736364:0:37867:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805e3 from it ffffaa914176bcb8 00010000:00010000:3.0:1769074804.736368:0:37867:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074804.736375:0:37867:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.745806:0:37869:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074804.745838:0:37869:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074804.745855:0:37869:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.745871:0:37869:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.745878:0:37869:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074804.745886:0:37869:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074804.745889:0:37869:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:1.0:1769074804.745896:0:37869:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa9140d7fb58 00010000:00010000:1.0:1769074804.745900:0:37869:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.745908:0:37869:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.745924:0:37869:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074804.745933:0:37869:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:1.0:1769074804.745942:0:37869:0:(file.c:6252:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffff8b100486d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074804.745946:0:37869:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348),name=f9.replay-single 00000002:00010000:1.0:1769074804.745957:0:37869:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074804.745964:0:37869:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.745972:0:37869:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.745978:0:37869:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074804.745982:0:37869:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) 00000080:00010000:1.0:1769074804.745984:0:37869:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffff8b100486d348) for lock 0x68a7113811d805e3 00000080:00010000:1.0:1769074804.745988:0:37869:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805e3 from it ffffaa9140d7fcb8 00010000:00010000:1.0:1769074804.745991:0:37869:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.745998:0:37869:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746028:0:37869:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074804.746035:0:37869:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074804.746040:0:37869:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746047:0:37869:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746053:0:37869:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074804.746057:0:37869:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074804.746060:0:37869:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:1.0:1769074804.746063:0:37869:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa9140d7fc10 00010000:00010000:1.0:1769074804.746066:0:37869:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746073:0:37869:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746080:0:37869:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074804.746087:0:37869:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=0 00000080:00200000:1.0:1769074804.746091:0:37869:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074804.746097:0:37869:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074804.746102:0:37869:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746108:0:37869:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746114:0:37869:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074804.746117:0:37869:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074804.746120:0:37869:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805dc 00000080:00010000:1.0:1769074804.746123:0:37869:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805dc from it ffffaa9140d7fd68 00010000:00010000:1.0:1769074804.746125:0:37869:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746132:0:37869:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746138:0:37869:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074804.746144:0:37869:0:(namei.c:2354:ll_unlink()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074804.746170:0:37869:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746193:0:37869:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074804.746203:0:37869:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x112:0x0]: losing layout lock 00010000:00010000:1.0:1769074804.746255:0:37869:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746263:0:37869:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746269:0:37869:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074804.746271:0:37869:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168e00/0x68a7113811d805e3 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30642ed expref: -99 pid: 37867 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074804.746289:0:37869:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029aca00/0x68a7113811d805dc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30642d1 expref: -99 pid: 37850 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074804.746311:0:37869:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074804.746323:0:37869:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006135c00 x1855007612425728/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074804.746348:0:37869:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37869:1855007612425728:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074804.746432:0:37869:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a3e240 going to sleep for 11 seconds 00000100:00100000:1.0:1769074804.767237:0:37869:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:37869:1855007612425728:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074804.767296:0:37869:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x112:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074804.767308:0:37869:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x112:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074804.767318:0:37869:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x112:0x0](ffff8b100486d348) 00010000:00010000:2.0:1769074805.279938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074806.307046:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074806.432219:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612425856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.432303:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612426112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.432333:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074806.432339:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612425984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.432360:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612426240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.452474:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612425984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.452492:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612426240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.452501:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612425856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074806.452511:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612426112:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074807.329048:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074808.368095:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074809.375839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074810.413027:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074811.427130:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074811.553191:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612426368:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.553298:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426624:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.553335:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074811.553342:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426496:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.553368:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.565499:0:2366:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 90194313217 gen 1 00000100:00100000:2.0:1769074811.565531:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426496:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.565548:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.565558:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612426368:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074811.565568:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612426624:192.168.204.102@tcp:400:kworker.0 00000001:02000400:2.0:1769074812.271747:0:38437:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 10: create |X| rename unlink ======= 04:40:10 (1769074810) 00010000:00010000:3.0:1769074812.450253:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074812.559773:0:38449:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074812.559807:0:38449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074812.559907:0:38449:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38449 initiator: MDT0 00010000:00010000:1.0:1769074812.559919:0:38449:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38449 initiator: MDT0 00010000:00010000:1.0:1769074812.559931:0:38449:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38449 initiator: MDT0 00000100:00100000:1.0:1769074812.559960:0:38449:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38449:1855007612426880:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:1.0:1769074812.560176:0:38449:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e26c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074812.565602:0:38449:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38449:1855007612426880:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:2.0:1769074812.565634:0:38449:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565670:0:38449:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565681:0:38449:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565691:0:38449:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565701:0:38449:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565708:0:38449:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565715:0:38449:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074812.565725:0:38449:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006135180 x1855007612426880/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074828 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074812.565738:0:38449:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565750:0:38449:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074812.565756:0:38449:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074812.565782:0:38449:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074812.565788:0:38449:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ea 00000080:00010000:2.0:1769074812.565792:0:38449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074812.565795:0:38449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ea from it ffffaa9141db3bd0 00010000:00010000:2.0:1769074812.565798:0:38449:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565805:0:38449:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074812.565825:0:38449:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074812.565853:0:38449:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074812.565858:0:38449:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074812.565870:0:38449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074812.565878:0:38449:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565888:0:38449:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074812.565894:0:38449:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074812.565898:0:38449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074812.565901:0:38449:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ea 00000080:00010000:2.0:1769074812.565905:0:38449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ea from it ffffaa9141db3cd0 00010000:00010000:2.0:1769074812.565908:0:38449:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565915:0:38449:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074812.565925:0:38449:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074812.565931:0:38449:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074812.565937:0:38449:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074812.565942:0:38449:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565949:0:38449:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074812.565956:0:38449:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074812.565959:0:38449:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074812.565962:0:38449:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805ea 00000080:00010000:2.0:1769074812.565965:0:38449:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ea from it ffffaa9141db3db0 00010000:00010000:2.0:1769074812.565968:0:38449:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.565975:0:38449:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074812.565983:0:38449:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074812.565987:0:38449:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074812.565991:0:38449:0:(namei.c:2020:ll_mknod()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074812.566015:0:38449:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.566056:0:38449:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074812.566067:0:38449:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074812.566069:0:38449:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d805ea lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064317 expref: -99 pid: 38449 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074812.566105:0:38449:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074812.566113:0:38449:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58a80 x1855007612427136/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074812.566136:0:38449:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38449:1855007612427136:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074812.566211:0:38449:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecc00 going to sleep for 11 seconds 00000100:00100000:3.0:1769074812.577860:0:38449:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38449:1855007612427136:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:3.0:1769074812.577916:0:38449:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x113:0x0](ffff8b100485ec08) 00000080:00200000:3.0:1769074812.577957:0:38449:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f10.replay-single, unhashed 1 00010000:00010000:3.0:1769074813.475037:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074813.709656:0:38458:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074813.709672:0:38458:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074813.709693:0:38458:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074813.709757:0:38458:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38458 initiator: MDT0 00010000:00010000:0.0:1769074813.709767:0:38458:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38458 initiator: MDT0 00010000:00010000:0.0:1769074813.709777:0:38458:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38458 initiator: MDT0 00000100:00100000:0.0:1769074813.709801:0:38458:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427264:192.168.204.102@tcp:101:lfs.0 00000100:00100000:0.0:1769074813.709896:0:38458:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3600 going to sleep for 11 seconds 00000100:00100000:1.0:1769074813.721307:0:38458:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427264:192.168.204.102@tcp:101:lfs.0 00010000:00010000:1.0:1769074813.721335:0:38458:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721348:0:38458:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721359:0:38458:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721371:0:38458:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721384:0:38458:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721392:0:38458:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721398:0:38458:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074813.721409:0:38458:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024a80 x1855007612427264/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074829 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074813.721421:0:38458:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721435:0:38458:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074813.721442:0:38458:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074813.721470:0:38458:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074813.721477:0:38458:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:1.0:1769074813.721482:0:38458:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074813.721485:0:38458:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3cb8 00010000:00010000:1.0:1769074813.721488:0:38458:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.721496:0:38458:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074813.722205:0:38458:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074813.722213:0:38458:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074813.722228:0:38458:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074813.722238:0:38458:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.722250:0:38458:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074813.722257:0:38458:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074813.722262:0:38458:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074813.722266:0:38458:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:1.0:1769074813.722270:0:38458:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3cb8 00010000:00010000:1.0:1769074813.722273:0:38458:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.722280:0:38458:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074813.722381:0:38458:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074813.722390:0:38458:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074813.722397:0:38458:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.722407:0:38458:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074813.722413:0:38458:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074813.722417:0:38458:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074813.722420:0:38458:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:1.0:1769074813.722424:0:38458:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3bb0 00010000:00010000:1.0:1769074813.722427:0:38458:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074813.722435:0:38458:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074813.722443:0:38458:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074813.722449:0:38458:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:1.0:1769074813.722533:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.722546:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00800000:00000004:1.0:1769074813.722553:0:38458:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1797, max_age 1805 00000100:00100000:1.0:1769074813.722587:0:38458:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427392:192.168.204.102@tcp:41:lfs.0 00000100:00100000:1.0:1769074813.722683:0:38458:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e2cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074813.725962:0:38458:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427392:192.168.204.102@tcp:41:lfs.0 00800000:00000004:1.0:1769074813.725992:0:38458:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:1.0:1769074813.726070:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726096:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726104:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726107:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726111:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726114:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726118:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726122:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726126:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726129:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726133:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726137:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726141:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726145:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726148:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726152:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726156:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726160:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726164:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726168:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726172:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726175:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726179:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726183:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726188:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726191:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726195:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726198:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726202:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726205:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726209:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726213:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726217:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726220:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726225:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726228:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726232:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726236:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726239:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726242:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726246:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726250:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726254:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726257:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726261:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726265:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726268:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726272:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726276:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726280:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726283:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726287:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726291:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726294:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726298:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726302:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726306:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726310:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726313:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726317:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726321:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726324:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726328:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726331:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726335:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726338:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726342:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726345:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726348:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726352:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726355:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726358:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726361:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726364:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726368:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726372:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726376:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726379:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726383:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726387:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726391:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726395:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726399:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726402:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726406:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726409:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726412:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726415:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726419:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726421:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726425:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726428:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726432:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726435:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726438:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726441:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726445:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726448:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726452:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726456:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726460:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726463:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726467:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726471:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726474:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726478:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726481:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726485:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726488:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726493:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726496:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726500:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726504:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726507:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726511:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726514:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726518:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726521:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726525:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726530:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726533:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726536:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726540:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726543:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726547:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726551:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726555:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00800000:00000080:1.0:1769074813.726558:0:38458:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.726562:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.726569:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00020000:00000004:1.0:1769074813.726576:0:38458:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1797, max_age 1805 00000100:00100000:1.0:1769074813.726611:0:38458:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427520:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074813.726708:0:38458:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e2cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074813.734302:0:38458:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427520:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074813.734333:0:38458:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074813.734410:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.734427:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00020000:00000004:1.0:1769074813.734434:0:38458:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1797, max_age 1805 00000100:00100000:1.0:1769074813.734470:0:38458:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427648:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074813.734534:0:38458:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e2cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074813.756873:0:38458:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38458:1855007612427648:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074813.756903:0:38458:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074813.756974:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.756991:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.756998:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757003:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757007:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757011:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757016:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757019:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757023:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757026:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757029:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757046:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757050:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757054:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757058:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757061:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757065:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757068:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757072:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757569:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757577:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757580:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757584:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757586:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757590:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757593:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757596:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757599:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757603:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757605:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757609:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757612:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757615:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757619:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757622:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757625:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757628:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757631:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757634:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757661:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757666:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757669:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757672:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757675:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757678:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757681:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757685:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757687:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757691:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757694:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757697:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757700:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757703:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757710:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757713:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757716:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757720:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757723:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757727:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757730:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757733:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757736:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757739:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757742:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757746:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757748:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757752:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757754:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757758:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757761:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757764:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757766:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757770:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757772:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757776:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757779:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757782:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757785:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757788:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757790:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757793:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757797:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757800:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757803:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757806:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757809:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757812:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757815:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757818:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757820:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757823:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757826:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757829:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757832:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757835:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757838:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757842:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757845:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757848:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757851:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757855:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757858:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757861:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757864:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757867:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757870:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757874:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757877:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757880:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757883:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757886:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757889:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757892:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757895:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757898:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757901:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757905:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757908:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757912:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757915:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757918:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757922:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757926:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757929:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200080:1.0:1769074813.757933:0:38458:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff638cf150 00020000:00000080:1.0:1769074813.757936:0:38458:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00000080:00200008:1.0:1769074813.757951:0:38458:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074813.837850:0:38465:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074813.837877:0:38465:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074813.837894:0:38465:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.837909:0:38465:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.837917:0:38465:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074813.837924:0:38465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074813.837927:0:38465:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:2.0:1769074813.837934:0:38465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3bd0 00010000:00010000:2.0:1769074813.837938:0:38465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.837946:0:38465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.837966:0:38465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074813.837977:0:38465:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074813.837980:0:38465:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074813.837985:0:38465:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074813.837990:0:38465:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.837997:0:38465:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.838003:0:38465:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074813.838006:0:38465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074813.838009:0:38465:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:2.0:1769074813.838011:0:38465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3cd0 00010000:00010000:2.0:1769074813.838014:0:38465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.838020:0:38465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.838025:0:38465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074813.838029:0:38465:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074813.838034:0:38465:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074813.838039:0:38465:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.838046:0:38465:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.838052:0:38465:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074813.838055:0:38465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074813.838058:0:38465:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f1 00000080:00010000:2.0:1769074813.838061:0:38465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f1 from it ffffaa9141db3db0 00010000:00010000:2.0:1769074813.838064:0:38465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.838084:0:38465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074813.838092:0:38465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074813.838096:0:38465:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074813.838100:0:38465:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074813.838123:0:38465:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.838172:0:38465:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074813.838179:0:38465:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074813.838183:0:38465:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac000/0x68a7113811d805f1 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064333 expref: -99 pid: 38458 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074813.838207:0:38465:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074813.838219:0:38465:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58700 x1855007612427904/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074813.838245:0:38465:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38465:1855007612427904:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074813.838340:0:38465:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074813.861457:0:38465:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:38465:1855007612427904:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074813.861509:0:38465:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x114:0x0](ffff8b1004869148) 00000080:00200000:2.0:1769074813.861550:0:38465:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:3.0:1769074813.915588:0:38468:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074813.915619:0:38468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074813.915966:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38468 initiator: MDT0 00010000:00010000:3.0:1769074813.915978:0:38468:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38468 initiator: MDT0 00010000:00010000:3.0:1769074813.915989:0:38468:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38468 initiator: MDT0 00000100:00100000:3.0:1769074813.916017:0:38468:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428032:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074813.916225:0:38468:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a52a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074813.919817:0:38468:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428032:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074813.919844:0:38468:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919863:0:38468:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919876:0:38468:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919888:0:38468:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919899:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919907:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919915:0:38468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074813.919938:0:38468:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037ddf80 x1855007612428032/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074829 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074813.919951:0:38468:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.919963:0:38468:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074813.919970:0:38468:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074813.919995:0:38468:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074813.920001:0:38468:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f8 00000080:00010000:3.0:1769074813.920006:0:38468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074813.920011:0:38468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f8 from it ffffaa9141d9bb58 00010000:00010000:3.0:1769074813.920014:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.920022:0:38468:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.920044:0:38468:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074813.920062:0:38468:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:3.0:1769074813.920066:0:38468:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:3.0:1769074813.920096:0:38468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074813.920139:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38468 initiator: MDT0 00010000:00010000:3.0:1769074813.920147:0:38468:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38468 initiator: MDT0 00010000:00010000:3.0:1769074813.920156:0:38468:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38468 initiator: MDT0 00000100:00100000:3.0:1769074813.920176:0:38468:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428160:192.168.204.102@tcp:101:rm.0 00000100:00100000:3.0:1769074813.920247:0:38468:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a52a80 going to sleep for 11 seconds 00000100:00100000:3.0:1769074813.939350:0:38468:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428160:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074813.939386:0:38468:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939410:0:38468:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939418:0:38468:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939431:0:38468:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939442:0:38468:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939454:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939462:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939469:0:38468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074813.939482:0:38468:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037ddf80 x1855007612428160/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074829 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074813.939494:0:38468:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939505:0:38468:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/276 (0 0) 00000080:00200000:3.0:1769074813.939511:0:38468:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074813.939532:0:38468:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x114:0x0], updating i_size 0 00000080:00200000:3.0:1769074813.939536:0:38468:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x114:0x0](ffff8b1004869148): rc = 0 00000080:00200000:3.0:1769074813.939551:0:38468:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x114:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074813.939557:0:38468:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x114:0x0](ffff8b1004869148) for lock 0x68a7113811d805ff 00000080:00010000:3.0:1769074813.939569:0:38468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x114:0x0](ffff8b1004869148) 00000080:00010000:3.0:1769074813.939573:0:38468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ff from it ffffaa9141d9bb20 00010000:00010000:3.0:1769074813.939576:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939585:0:38468:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074813.939625:0:38468:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x114:0x0](ffff8b1004869148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074813.939690:0:38468:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x114:0x0](ffff8b1004869148),name=fsa-oleg402-client.virtnet 00000002:00010000:3.0:1769074813.939720:0:38468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x114:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074813.939729:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939740:0:38468:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939747:0:38468:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074813.939752:0:38468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x114:0x0](ffff8b1004869148) 00000080:00010000:3.0:1769074813.939754:0:38468:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x114:0x0](ffff8b1004869148) for lock 0x68a7113811d805ff 00000080:00010000:3.0:1769074813.939758:0:38468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805ff from it ffffaa9141d9bcb8 00010000:00010000:3.0:1769074813.939761:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939768:0:38468:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939829:0:38468:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074813.939837:0:38468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074813.939844:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939853:0:38468:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939860:0:38468:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074813.939864:0:38468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074813.939867:0:38468:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f8 00000080:00010000:3.0:1769074813.939871:0:38468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f8 from it ffffaa9141d9bc10 00010000:00010000:3.0:1769074813.939874:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939882:0:38468:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939890:0:38468:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074813.939899:0:38468:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:3.0:1769074813.939904:0:38468:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074813.939911:0:38468:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074813.939917:0:38468:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939924:0:38468:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939930:0:38468:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074813.939934:0:38468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074813.939936:0:38468:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d805f8 00000080:00010000:3.0:1769074813.939940:0:38468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d805f8 from it ffffaa9141d9bd68 00010000:00010000:3.0:1769074813.939942:0:38468:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.939949:0:38468:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.939956:0:38468:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074813.939962:0:38468:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074813.939982:0:38468:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.940005:0:38468:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074813.940016:0:38468:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x114:0x0]: losing layout lock 00010000:00010000:3.0:1769074813.940047:0:38468:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.940055:0:38468:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.940060:0:38468:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074813.940063:0:38468:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168600/0x68a7113811d805ff lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x114:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306436b expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074813.940092:0:38468:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169e00/0x68a7113811d805f8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306434f expref: -99 pid: 38468 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074813.940110:0:38468:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074813.940118:0:38468:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037ddf80 x1855007612428416/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074813.940146:0:38468:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428416:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074813.940215:0:38468:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a52a80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074813.951203:0:38468:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38468:1855007612428416:192.168.204.102@tcp:36:rm.0 00000080:00200000:0.0:1769074813.951265:0:38468:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x114:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074813.951275:0:38468:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x114:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074813.951283:0:38468:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x114:0x0](ffff8b1004869148) 00000100:00100000:1.0:1769074814.111948:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428544:192.168.204.102@tcp:17:kworker.0 00000100:00100000:1.0:1769074814.116238:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428544:192.168.204.102@tcp:17:kworker.0 00010000:00010000:1.0:1769074814.496956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074815.520243:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074816.547272:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074816.674309:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074816.674406:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612429056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074816.674439:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074816.674446:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612428672:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074816.674478:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074816.684346:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612428672:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074816.684381:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074816.684394:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612428800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074816.689164:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612429056:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074817.567956:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074818.361777:0:38499:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1769074818.404664:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074818.404703:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.404783:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38500 initiator: MDT0 00010000:00010000:2.0:1769074818.404796:0:38500:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00010000:00010000:2.0:1769074818.404807:0:38500:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00000100:00100000:2.0:1769074818.404837:0:38500:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429184:192.168.204.102@tcp:101:mv.0 00000100:00100000:2.0:1769074818.404955:0:38500:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074818.420197:0:38500:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429184:192.168.204.102@tcp:101:mv.0 00010000:00010000:2.0:1769074818.420238:0:38500:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420253:0:38500:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420269:0:38500:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420288:0:38500:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420303:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420311:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420319:0:38500:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074818.420332:0:38500:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5a680 x1855007612429184/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074818.420346:0:38500:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420361:0:38500:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074818.420449:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074818.420485:0:38500:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074818.420492:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:2.0:1769074818.420499:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074818.420502:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bbd0 00010000:00010000:2.0:1769074818.420506:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420515:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.420546:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074818.420579:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074818.420596:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.420604:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420615:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.420622:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074818.420625:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074818.420628:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:2.0:1769074818.420631:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bbd0 00010000:00010000:2.0:1769074818.420634:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.420686:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.420696:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074818.420715:0:38500:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=0 00000080:00200000:2.0:1769074818.420720:0:38500:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074818.420735:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f10.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.420779:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38500 initiator: MDT0 00010000:00010000:2.0:1769074818.420786:0:38500:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00010000:00010000:2.0:1769074818.420797:0:38500:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00000100:00100000:2.0:1769074818.420821:0:38500:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429312:192.168.204.102@tcp:101:mv.0 00000100:00100000:2.0:1769074818.421085:0:38500:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074818.429818:0:38500:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429312:192.168.204.102@tcp:101:mv.0 00010000:00010000:1.0:1769074818.429845:0:38500:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429870:0:38500:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429879:0:38500:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429891:0:38500:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429901:0:38500:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429914:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429921:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429933:0:38500:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074818.429956:0:38500:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5a680 x1855007612429312/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074818.429970:0:38500:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.429979:0:38500:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/275 (0 0) 00000080:00200000:1.0:1769074818.429987:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074818.430017:0:38500:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x113:0x0], updating i_size 0 00000080:00200000:1.0:1769074818.430020:0:38500:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x113:0x0](ffff8b100485ec08): rc = 0 00000080:00200000:1.0:1769074818.430036:0:38500:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x113:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1769074818.430042:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) for lock 0x68a7113811d8060d 00000080:00010000:1.0:1769074818.430051:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) 00000080:00010000:1.0:1769074818.430055:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8060d from it ffffaa914176bdb8 00010000:00010000:1.0:1769074818.430058:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.430067:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.430104:0:38500:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=2048 00000080:00200000:1.0:1769074818.430109:0:38500:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f10.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:1.0:1769074818.430128:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f10.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.430170:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.430177:0:38500:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.430185:0:38500:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00000100:00100000:1.0:1769074818.430204:0:38500:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429440:192.168.204.102@tcp:101:mv.0 00000100:00100000:1.0:1769074818.430289:0:38500:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e2000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074818.439163:0:38500:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429440:192.168.204.102@tcp:101:mv.0 00010000:00010000:1.0:1769074818.439186:0:38500:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439201:0:38500:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439208:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439215:0:38500:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439223:0:38500:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439230:0:38500:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b10060b8600 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:1.0:1769074818.439235:0:38500:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439242:0:38500:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439248:0:38500:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439256:0:38500:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074818.439261:0:38500:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 38500 initiator: MDT0 00010000:00010000:1.0:1769074818.439267:0:38500:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8600/0x68a7113811d80614 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 38500 initiator: MDT0 00000002:00100000:1.0:1769074818.439283:0:38500:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1006136680 x1855007612429440/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'mv.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074818.439297:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074818.439321:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439334:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074818.439344:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bbd8 00010000:00010000:1.0:1769074818.439348:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439356:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.439391:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.439406:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.439413:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439421:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.439428:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.439432:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.439436:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.439440:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bc98 00010000:00010000:1.0:1769074818.439443:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439451:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.439459:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074818.439465:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.439541:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.439548:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439902:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.439911:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.439917:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.439920:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.439924:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bcc8 00010000:00010000:1.0:1769074818.439927:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.439935:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.439954:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074818.440005:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.440012:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.440018:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440026:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440032:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.440035:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.440038:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.440041:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bb58 00010000:00010000:1.0:1769074818.440044:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440051:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440058:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074818.440063:0:38500:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single-2, flags=8 00000080:00200000:1.0:1769074818.440096:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.440102:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.440107:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440114:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440126:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.440130:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.440133:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.440138:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bbd0 00010000:00010000:1.0:1769074818.440142:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440150:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440158:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074818.440164:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.440168:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.440172:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440178:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440184:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.440187:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.440189:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.440192:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bbd0 00010000:00010000:1.0:1769074818.440195:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440202:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440209:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074818.440214:0:38500:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single, flags=0 00000080:00200000:1.0:1769074818.440216:0:38500:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single-2, flags=2048 00000080:00200000:1.0:1769074818.440219:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.440223:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.440227:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440233:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440239:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.440242:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.440244:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.440247:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bc98 00010000:00010000:1.0:1769074818.440249:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440256:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440262:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074818.440265:0:38500:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074818.440268:0:38500:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074818.440272:0:38500:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440279:0:38500:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440285:0:38500:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074818.440288:0:38500:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074818.440290:0:38500:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80606 00000080:00010000:1.0:1769074818.440293:0:38500:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80606 from it ffffaa914176bcc8 00010000:00010000:1.0:1769074818.440295:0:38500:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440301:0:38500:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440307:0:38500:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074818.440311:0:38500:0:(namei.c:2433:ll_rename()) VFS Op:oldname=f10.replay-single, src_dir=[0x200000007:0x1:0x0](ffff8b1018194b08), newname=f10.replay-single-2, tgt_dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074818.440331:0:38500:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440352:0:38500:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074818.440365:0:38500:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x113:0x0]: losing layout lock 00010000:00010000:1.0:1769074818.440397:0:38500:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440405:0:38500:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440410:0:38500:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074818.440413:0:38500:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8060d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30643b1 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074818.440433:0:38500:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699c00/0x68a7113811d80606 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064395 expref: -99 pid: 38500 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074818.440450:0:38500:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074818.440458:0:38500:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006136680 x1855007612429696/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074818.440490:0:38500:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429696:192.168.204.102@tcp:36:mv.0 00000100:00100000:1.0:1769074818.440571:0:38500:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e2000 going to sleep for 11 seconds 00000100:00100000:1.0:1769074818.451196:0:38500:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:38500:1855007612429696:192.168.204.102@tcp:36:mv.0 00000080:00200000:2.0:1769074818.464974:0:38501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074818.466524:0:38501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.466666:0:38501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.466681:0:38501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.466693:0:38501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38501 initiator: MDT0 00000100:00100000:2.0:1769074818.466729:0:38501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38501:1855007612429824:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074818.466844:0:38501:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074818.470267:0:38501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38501:1855007612429824:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074818.470311:0:38501:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470326:0:38501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470340:0:38501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470352:0:38501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470372:0:38501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470381:0:38501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470388:0:38501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074818.470399:0:38501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59180 x1855007612429824/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074818.470414:0:38501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470427:0:38501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074818.470434:0:38501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074818.470468:0:38501:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074818.470475:0:38501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8061b 00000080:00010000:2.0:1769074818.470481:0:38501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074818.470485:0:38501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9141c93b58 00010000:00010000:2.0:1769074818.470488:0:38501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.470497:0:38501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.470526:0:38501:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074818.470552:0:38501:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074818.470558:0:38501:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074818.470587:0:38501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f10.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.470675:0:38501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.470682:0:38501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.470693:0:38501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38501 initiator: MDT0 00000100:00100000:2.0:1769074818.470720:0:38501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38501:1855007612429952:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074818.470807:0:38501:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ecb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074818.484912:0:38501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:38501:1855007612429952:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074818.484952:0:38501:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.484969:0:38501:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.484977:0:38501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.484984:0:38501:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.484993:0:38501:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.485011:0:38501:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698400 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074818.485017:0:38501:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.485025:0:38501:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.485032:0:38501:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.485040:0:38501:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074818.485047:0:38501:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 38501 initiator: MDT0 00010000:00010000:2.0:1769074818.485053:0:38501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80622 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 38501 initiator: MDT0 00000002:00100000:2.0:1769074818.485076:0:38501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1005d59180 x1855007612429952/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074818.485094:0:38501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074818.485130:0:38501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.485145:0:38501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074818.485156:0:38501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9141c93940 00010000:00010000:2.0:1769074818.485160:0:38501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.485168:0:38501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.485272:0:38501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074818.485290:0:38501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074818.485296:0:38501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.485304:0:38501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.485310:0:38501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074818.485315:0:38501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074818.485319:0:38501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8061b 00000080:00010000:2.0:1769074818.485324:0:38501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9141c93c10 00010000:00010000:2.0:1769074818.485327:0:38501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074818.485333:0:38501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074818.485341:0:38501:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074818.485353:0:38501:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single, flags=0 00010000:00010000:1.0:1769074818.598043:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074819.615932:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074820.639889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074821.665083:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074821.792045:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612430080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074821.792045:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612430208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074821.792202:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612430464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074821.796186:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074821.807363:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074821.815279:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612430464:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074822.687908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074823.711925:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074824.736735:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074825.759901:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074826.791213:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074826.912392:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612430592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074826.912483:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074826.912517:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074826.912524:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074826.912564:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430976:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074826.918780:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430976:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074826.918815:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430848:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074827.812809:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074828.832033:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074829.856685:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074830.879844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074831.904256:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074832.034272:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612431104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074832.034397:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431360:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074832.034435:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074832.034442:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431232:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074832.034468:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431488:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074832.046181:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431360:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074832.057680:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431488:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074832.927906:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074833.956932:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074834.975881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074836.002999:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074837.024447:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074837.151921:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074821/real 1769074821] req@ffff8b1006136a00 x1855007612430080/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074837.153386:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:0.0:1769074837.153393:0:37:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff8b101127b800: disconnect after 24s idle 00000100:00100000:2.0:1769074837.153475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612431872:192.168.204.102@tcp:9:kworker.0 00000100:00080000:0.0:1769074837.153703:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:0.0:1769074837.153707:0:37:0:(import.c:2080:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff8b101127b800: disconnect after 24s idle 00000100:00100000:2.0:1769074837.160892:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432000:192.168.204.102@tcp:9:kworker.0 00000100:00100000:0.0:1769074837.164573:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612431744:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074837.164694:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074837.164705:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612431616:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074837.171884:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006027480 x1855007612432000/t0(0) o9->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074847 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074837.171899:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1769074837.171903:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432000:192.168.204.102@tcp:9:kworker.0 00000100:00080000:2.0:1769074837.171918:0:2363:0:(import.c:1990:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff8b1006024e00 x1855007612431872/t0(0) o9->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 0 dl 1769074847 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074837.171925:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:2.0:1769074837.171928:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612431872:192.168.204.102@tcp:9:kworker.0 00000100:00100000:1.0:1769074837.208970:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006136a00 x1855007612430080/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074837.208993:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1769074837.208998:0:2365:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1769074837.209003:0:2365:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1769074837.209005:0:2365:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1769074837.209011:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006136a00 x1855007612430080/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074837.209022:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006027b80 x1855007612430592/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074842 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074837.209032:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006026d80 x1855007612431104/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074848 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074837.209040:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006025f80 x1855007612431616/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074853 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1769074837.209055:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1769074837.209059:0:2365:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1769074837.209061:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1769074837.209064:0:2365:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:2.0:1769074837.209196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074837.209207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769074837.209211:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074837.209221:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1769 00000100:00080000:2.0:1769074837.209238:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769074837.209278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612432128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074837.209338:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612431616:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074837.209395:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612431104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074837.209404:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612430592:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074837.209408:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612430080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074837.209555:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612432128:192.168.204.102@tcp:250:kworker.0 00010000:00010000:2.0:1769074838.056314:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074838.177611:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074821/real 1769074821] req@ffff8b1006137100 x1855007612430208/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074838.177681:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006137100 x1855007612430208/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074838.177691:0:2364:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074838.177695:0:2364:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074838.177701:0:2364:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00080000:3.0:1769074838.188354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074838.188364:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074838.188367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074838.188370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074838.188375:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074838.188383:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1789 00000100:00080000:3.0:1769074838.188399:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074838.188411:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074838.188445:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612432256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074838.188551:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074838.188853:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612430208:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074839.074283:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074840.096307:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074841.120009:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074842.143987:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074842.208000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074842.208014:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769074842.208017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074842.208021:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:1.0:1769074842.337332:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074826/real 1769074826] req@ffff8b1006024700 x1855007612430720/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074842.337378:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006024700 x1855007612430720/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074842.337389:0:2365:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 62, was 61): CONNECTING 00000100:00100000:1.0:1769074842.337425:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612430720:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074843.167851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074843.233168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074843.233181:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074843.233186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074843.233190:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1769074844.192124:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074845.216310:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074846.241359:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074847.263966:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074847.329258:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074837/real 1769074837] req@ffff8b1005d59180 x1855007612432128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074847 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074847.329305:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612432128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074847 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074847.329331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769074847.329365:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769074847.329370:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612432128:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074847.330887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074847.330900:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769074847.330905:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074847.330917:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1830 00000100:00080000:3.0:1769074847.330943:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769074847.330952:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769074847.331018:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612432384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769074847.331056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074847.331061:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074847.335840:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612432384:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769074847.350472:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:2.0:1769074847.350495:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1769074847.350505:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3063f43 to 0x143201ebf3064428 00000100:00080000:2.0:1769074847.350521:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1769074847.350529:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:2.0:1769074847.352526:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612432384:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769074847.352561:0:38690:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:0.0:1769074847.352568:0:38690:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:0.0:1769074847.352573:0:38690:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:0.0:1769074847.352594:0:38690:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074847.352606:0:38690:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074847.352612:0:38690:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074847.352633:0:38690:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074847.352672:0:38690:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029ac800/0x68a7113811d805c0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063f51 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.352692:0:38690:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074847.352698:0:38690:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074847.352703:0:38690:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074847.352707:0:38690:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074847.352710:0:38690:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10060b8e00/0x68a7113811d805ce lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063f5f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.352724:0:38690:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074847.352731:0:38690:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074847.352736:0:38690:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:0.0:1769074847.352739:0:38690:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074847.352743:0:38690:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169400/0x68a7113811d805d5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf30640d2 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.352755:0:38690:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:0.0:1769074847.352760:0:38690:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074847.352765:0:38690:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:0.0:1769074847.352768:0:38690:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074847.352771:0:38690:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10060b9c00/0x68a7113811d805c7 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3063f58 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:0.0:1769074847.352788:0:38690:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:0.0:1769074847.352795:0:38690:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:0.0:1769074847.352797:0:38690:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:0.0:1769074847.352800:0:38690:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:0.0:1769074847.352804:0:38690:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:3.0:1769074848.288205:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074848.355219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074838/real 1769074838] req@ffff8b10037dd180 x1855007612432256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074848 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074848.355268:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612432256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074848 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074848.355293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074848.355326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074848.355331:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432256:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769074848.355508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074848.355513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769074848.355517:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769074848.355523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1831 00000100:00080000:3.0:1769074848.355536:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769074848.355543:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769074848.355558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074848.355577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612432640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074848.355688:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769074848.356563:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612432512:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074848.375231:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612432512:192.168.204.102@tcp:400:kworker.0 00000100:00000400:0.0:1769074848.479855:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074832/real 1769074832] req@ffff8b1006025c00 x1855007612431232/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074848 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074848.479907:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006025c00 x1855007612431232/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074848 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074848.479916:0:2365:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 63, was 61): CONNECTING 00000100:00100000:0.0:1769074848.479940:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612431232:192.168.204.102@tcp:400:kworker.0 10000000:01000000:2.0:1769074849.247803:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:2.0:1769074849.247814:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 27 10000000:01000000:2.0:1769074849.247822:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1769074849.247910:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.247924:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.247937:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.247980:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612432768:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.248112:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.251982:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612432768:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074849.252022:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.252037:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.252148:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612432896:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.252264:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00010000:00010000:3.0:1769074849.327972:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074849.331516:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612432896:192.168.204.102@tcp:256:ll_cfg_requeue.0 10000000:01000000:2.0:1769074849.331584:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1769074849.331598:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.331612:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074849.331624:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1769074849.331627:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:2.0:1769074849.331630:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1769074849.331742:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.331751:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.331762:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.331789:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433024:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.331898:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.337876:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433024:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074849.337934:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.337950:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.338080:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433152:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.338212:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.349960:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433152:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.350073:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433280:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.350240:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.363385:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433280:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1769074849.365513:0:38691:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:2.0:1769074849.367259:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:2.0:1769074849.367303:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1769074849.367318:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.367334:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074849.367348:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1769074849.367351:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:2.0:1769074849.367355:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1769074849.367416:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.367426:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.367438:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.367490:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433408:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.367657:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.373832:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433408:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074849.373877:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.373892:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.374404:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433536:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.374593:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.378591:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433536:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.378710:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433664:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.378846:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.382876:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433664:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1769074849.384531:0:38692:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:2.0:1769074849.385006:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:2.0:1769074849.385034:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1769074849.385218:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.385233:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074849.385246:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1769074849.385249:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:2.0:1769074849.385252:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1769074849.385305:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.385314:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.385326:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.385359:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433792:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.385546:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.390562:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433792:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074849.390609:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.390627:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074849.390798:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433920:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074849.394899:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074849.395189:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612433920:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1769074849.395249:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1769074849.395261:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074849.395273:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074850.337969:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074851.360198:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074852.389378:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074852.541338:0:2356:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.102@tcp (source: 12345-192.168.204.102@tcp) 00000100:00100000:1.0:1769074852.541606:0:13988:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007624756992 00000100:00100000:1.0:1769074852.541621:0:13988:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007624756992 00000100:00100000:1.0:1769074852.541737:0:13988:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.102@tcp, seq: 200 00000100:00100000:1.0:1769074852.541748:0:13988:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff8b1006025c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:41996:x1855007624756992:12345-192.168.204.102@tcp:104: 00010000:00010000:1.0:1769074852.541770:0:13988:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074852.541873:0:13988:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074852.541881:0:13988:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff8b1006025c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:86724556-597d-4d0e-ac75-6d9d64145050+3:41996:x1855007624756992:12345-192.168.204.102@tcp:104: Request processed in 135us (558us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074852.541888:0:13988:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.102@tcp, seq: 200 00010000:00010000:3.0:1769074852.543993:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074852.544294:0:13997:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074852.544310:0:13997:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698000 already unused, calling callback (ffffffffc121fc20) 10000000:00010000:3.0:1769074852.544316:0:13997:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074852.544328:0:13997:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074852.544335:0:13997:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074852.544342:0:13997:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074852.544446:0:13997:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074852.544453:0:13997:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074852.544489:0:13997:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074852.544496:0:13997:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698000/0x68a7113811d80629 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x143201ebf306442f expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074852.544518:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074852.544520:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074852.544523:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074852.546385:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612434048:192.168.204.102@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074852.550183:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612434048:192.168.204.102@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074853.409605:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074853.535833:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074837/real 1769074837] req@ffff8b1006024380 x1855007612431744/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074853 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074853.535885:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006024380 x1855007612431744/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074853 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074853.535906:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 63, was 61): CONNECTING 00000100:00100000:1.0:1769074853.535927:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612431744:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074853.536519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074853.536526:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769074853.536747:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434176:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074853.548322:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434176:192.168.204.102@tcp:400:kworker.0 10000000:01000000:3.0:1769074853.730070:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1769074853.730083:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 27 10000000:01000000:3.0:1769074853.730104:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1769074853.730189:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074853.730204:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074853.730219:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074853.730271:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612434304:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074853.730799:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a52840 going to sleep for 11 seconds 00000100:00100000:3.0:1769074853.761612:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612434304:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074853.761671:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074853.761686:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074853.761774:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612434432:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1769074853.761890:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1010a52840 going to sleep for 11 seconds 00000100:00100000:3.0:1769074853.795410:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612434432:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1769074853.795484:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1769074853.795681:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074853.795694:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074853.795707:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:2.0:1769074853.826813:0:38704:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 63, was 0): CONNECTING 00010000:00080000:2.0:1769074853.826846:0:38704:0:(ldlm_lib.c:85:import_set_conn()) imp ffff8b1008679800@lustre-MDT0000-mdc-ffff8b101127b800: found existing conn 192.168.204.102@tcp, moved to head 00010000:00010000:1.0:1769074854.431851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074855.456778:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074856.480393:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074857.503955:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074858.530731:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769074858.655958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074858.655967:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074858.655993:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074858.656021:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074858.669604:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434560:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074859.574523:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074860.577418:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074861.601114:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074862.625128:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074862.707120:0:38793:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074862.707562:0:38793:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074862.707588:0:38793:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074862.707606:0:38793:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.707622:0:38793:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074862.707630:0:38793:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074862.712843:0:38793:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074862.712854:0:38793:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8061b 00000080:00010000:3.0:1769074862.712867:0:38793:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9140dd3cb8 00010000:00010000:3.0:1769074862.712874:0:38793:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.712887:0:38793:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074862.713634:0:38793:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074862.713967:0:38793:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074862.713987:0:38793:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074862.714004:0:38793:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.714018:0:38793:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074862.714026:0:38793:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074862.714034:0:38793:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074862.714037:0:38793:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8061b 00000080:00010000:3.0:1769074862.714043:0:38793:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9140dd3cb8 00010000:00010000:3.0:1769074862.714047:0:38793:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.714055:0:38793:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074862.714178:0:38793:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074862.714186:0:38793:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074862.714192:0:38793:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.714200:0:38793:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074862.714206:0:38793:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074862.714210:0:38793:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074862.714214:0:38793:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8061b 00000080:00010000:3.0:1769074862.714217:0:38793:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8061b from it ffffaa9140dd3bb0 00010000:00010000:3.0:1769074862.714221:0:38793:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074862.714229:0:38793:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074862.714242:0:38793:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074862.714248:0:38793:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074862.714548:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:3.0:1769074862.714564:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00800000:00000004:3.0:1769074862.714571:0:38793:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1806, max_age 1854 00000100:00080000:3.0:1769074862.714614:0:38793:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b10037dd880 x1855007612434688/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074862.714625:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1014164f00 going to sleep for 0 seconds 00010000:00010000:3.0:1769074863.647996:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074863.775759:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1014164f00 going to sleep for 0 seconds 00000100:00100000:2.0:1769074863.775949:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074848/real 1769074848] req@ffff8b10037ded80 x1855007612432640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074863 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074863.775988:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612432640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074863 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074863.776012:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074863.776041:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074863.776047:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612432640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074863.776325:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434816:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074863.778364:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074863.778374:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074863.778378:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074863.778388:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 0 00000100:00080000:0.0:1769074863.778403:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074863.778424:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074863.778884:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612434944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074863.779259:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612434944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074863.792924:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612434816:192.168.204.102@tcp:400:kworker.0 00010000:00080000:3.0:1769074863.794013:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1006024a80 x1855007612434944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074873 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074863.794030:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 90194313220 gen 1 00000100:00100000:3.0:1769074863.794056:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136680 x1855007612429696/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/456 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074863.794072:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 22 00000100:00080000:3.0:1769074863.794094:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1769074863.794101:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074863.794104:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 90194313220 gen 1 00000100:00100000:3.0:1769074863.794108:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136680 x1855007612429696/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/456 e 0 to 0 dl 1769074834 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074863.794116:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 90194313220 last 0 00000100:00080000:3.0:1769074863.794124:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1006136680 x1855007612429696/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/456 e 0 to 0 dl 1769074834 ref 1 fl New:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074863.794142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612434944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769074863.794164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612429696:192.168.204.102@tcp:36:mv.0 00000100:00080000:3.0:1769074863.886809:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1006136680 x1855007612429696/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/456 e 0 to 0 dl 1769074879 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074863.886830:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074863.886833:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 90194313220 gen 1 00000100:00100000:3.0:1769074863.886839:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136680 x1855007612429696/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/456 e 0 to 0 dl 1769074879 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074863.886848:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 90194313220 last 90194313221 00000100:00080000:3.0:1769074863.886853:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:3.0:1769074863.889992:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612429696:192.168.204.102@tcp:36:mv.0 00010000:00080000:1.0:1769074863.890087:0:38808:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:1.0:1769074863.890093:0:38808:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (1) 00010000:00010000:1.0:1769074863.890146:0:38808:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d8061b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf30643f0 expref: -99 pid: 38501 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074863.890162:0:38808:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00000100:00080000:1.0:1769074863.890624:0:38808:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:2.0:1769074863.890931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435072:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:2.0:1769074863.989264:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 90194313221 gen 1 00000100:00080000:2.0:1769074863.989293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:2.0:1769074863.989306:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1006137100 x1855007612435072/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074879 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074863.989315:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1006137100 x1855007612435072/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074879 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074863.989323:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:3.0:1769074863.989753:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:3.0:1769074863.989762:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000d8c59ca1): req@ffff8b10037dd880 x1855007612434688/t0(0) o41->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074863.989789:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435072:192.168.204.102@tcp:400:ldlm_lock_repla.0 00000100:00100000:0.0:1769074863.990164:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1014164f00 going to sleep for 11 seconds 00000100:00100000:0.0:1769074864.014277:0:38793:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38793:1855007612434688:192.168.204.102@tcp:41:lfs.0 00800000:00000004:0.0:1769074864.014308:0:38793:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:0.0:1769074864.014395:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014422:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014430:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014434:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014438:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014442:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014445:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014448:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014451:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014454:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014458:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014461:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014464:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014467:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014471:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014474:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014477:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014480:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014484:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014487:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014490:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014493:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014497:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014500:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014503:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014507:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014510:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014513:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014516:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014520:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014523:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014527:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014531:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014535:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014538:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014541:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014545:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014548:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014552:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014556:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014561:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014564:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014567:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014571:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014575:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014578:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014582:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014586:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014590:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014593:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014597:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014600:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014604:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014607:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014610:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014613:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014616:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014620:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014623:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014627:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014631:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014634:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014675:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014681:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014686:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014689:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014693:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014696:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014701:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014704:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014708:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014712:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014716:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014719:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014723:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014727:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014730:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014734:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014738:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014741:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014745:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014748:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014752:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014755:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014759:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014762:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014765:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014768:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014773:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014777:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014780:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014784:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014787:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014791:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014794:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014798:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014802:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014804:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014808:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014811:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014814:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014817:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014821:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014824:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014827:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014830:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014833:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014836:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014840:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014843:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014847:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014850:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014854:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014869:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014875:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014879:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014883:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014887:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014891:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014895:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014899:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.014902:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.014906:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.015086:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.015096:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.015101:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.015105:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00800000:00000080:0.0:1769074864.015108:0:38793:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00000080:00200080:0.0:1769074864.015112:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:0.0:1769074864.015117:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00020000:00000004:0.0:1769074864.015125:0:38793:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1806, max_age 1856 00000100:00080000:0.0:1769074864.015142:0:38793:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1789 00000100:00080000:0.0:1769074864.015154:0:38793:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074864.015165:0:38793:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:0.0:1769074864.015190:0:38793:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612435200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074864.015249:0:38793:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1006024000 x1855007612435328/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074864.015259:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 0 seconds 00000100:00100000:2.0:1769074864.015790:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435200:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074864.029596:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934747 gen 10 00000100:00080000:2.0:1769074864.029613:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff8b101127b800: connect to target with instance 2 00000100:00080000:2.0:1769074864.029622:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1769074864.029634:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074864.029668:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074864.029674:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:2.0:1769074864.029677:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1769074864.029684:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074864.029686:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074864.029700:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000f3e44fc1): req@ffff8b1006024000 x1855007612435328/t0(0) o13->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074864.029728:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435200:192.168.204.102@tcp:8:lfs.0 00000100:00100000:0.0:1769074864.030255:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074864.038723:0:38793:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38793:1855007612435328:192.168.204.102@tcp:13:lfs.0 00020000:00000004:0.0:1769074864.038751:0:38793:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:0.0:1769074864.038826:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:0.0:1769074864.038843:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602a800 uarg=0000000000000000 00020000:00000004:0.0:1769074864.038850:0:38793:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1806, max_age 1856 00000100:00080000:0.0:1769074864.038870:0:38793:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1789 00000100:00080000:0.0:1769074864.038881:0:38793:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074864.038890:0:38793:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 3407872, lost: 0. 00000100:00100000:0.0:1769074864.038911:0:38793:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612435456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074864.038952:0:38793:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1006024000 x1855007612435584/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074864.038962:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 0 seconds 00000100:00100000:2.0:1769074864.039264:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435456:192.168.204.102@tcp:8:lfs.0 00000100:00100000:2.0:1769074864.045599:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967450 gen 10 00000100:00080000:2.0:1769074864.045617:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff8b101127b800: connect to target with instance 1 00000100:00080000:2.0:1769074864.045626:0:2363:0:(import.c:951:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff8b101127b800: Resetting ns_connect_flags to server flags: 0xa0425af2e3440038 00000080:00000004:2.0:1769074864.045636:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074864.045668:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074864.045672:0:2363:0:(import.c:1204:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:2.0:1769074864.045677:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00000080:00000004:2.0:1769074864.045684:0:2363:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1769074864.045686:0:2363:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1769074864.045699:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000f3e44fc1): req@ffff8b1006024000 x1855007612435584/t0(0) o13->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074864.045726:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612435456:192.168.204.102@tcp:8:lfs.0 00000100:00100000:0.0:1769074864.048840:0:38793:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 11 seconds 00000100:00100000:3.0:1769074864.095036:0:38793:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:38793:1855007612435584:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074864.095067:0:38793:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:2.0:1769074864.095321:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095339:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095347:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095351:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095356:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095360:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095363:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095367:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095371:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095375:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095378:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095382:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095385:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095388:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095392:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095395:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095399:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095401:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095405:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095866:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095874:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095878:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095883:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095886:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095890:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095892:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095896:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095898:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095902:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095904:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.095907:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.095994:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096000:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096004:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096008:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096012:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096015:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096018:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096022:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096024:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096028:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096032:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096035:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096038:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096042:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096045:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096049:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096052:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096055:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096058:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096062:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096065:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096069:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096073:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096077:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096080:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096084:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096088:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096091:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096095:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096099:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096103:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096106:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096110:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096114:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096118:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096121:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096124:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096128:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096131:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096135:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096138:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096142:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096146:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096150:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096153:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096157:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096160:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096164:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096167:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096170:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096174:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096177:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096180:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096184:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096188:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096192:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096194:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096198:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096201:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096205:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096208:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096212:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096215:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096219:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096221:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096225:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096239:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096245:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096250:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096254:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096258:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096262:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096265:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096269:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096272:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096276:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096280:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096283:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096287:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096290:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096294:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096297:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096300:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096303:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096306:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096310:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096313:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096317:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096320:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096323:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096327:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096330:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096333:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200080:2.0:1769074864.096337:0:38793:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffc4cb44bf0 00020000:00000080:2.0:1769074864.096340:0:38793:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa400 uarg=0000000000000000 00000080:00200008:2.0:1769074864.096352:0:38793:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:3.0:1769074864.672888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074865.696024:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074866.724530:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074867.744787:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1769074868.622918:0:39045:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:1.0:1769074868.772808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074868.904490:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612435840:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.904579:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612436096:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.904611:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1769074868.904617:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612435712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.905023:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612435968:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.936958:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612435712:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.936982:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612435840:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.936995:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612436096:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074868.937103:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612435968:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074869.806744:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074870.574343:0:39199:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:0.0:1769074870.818145:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074871.839929:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1769074872.823184:0:39237:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074872.823199:0:39237:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074872.823221:0:39237:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074872.823281:0:39237:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39237 initiator: MDT0 00010000:00010000:1.0:1769074872.823291:0:39237:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39237 initiator: MDT0 00010000:00010000:1.0:1769074872.823301:0:39237:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39237 initiator: MDT0 00000100:00100000:1.0:1769074872.823326:0:39237:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436224:192.168.204.102@tcp:101:lfs.0 00000100:00100000:1.0:1769074872.823427:0:39237:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e20c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074872.830137:0:39237:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436224:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074872.830164:0:39237:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830177:0:39237:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830189:0:39237:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830200:0:39237:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830211:0:39237:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830220:0:39237:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830227:0:39237:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074872.830239:0:39237:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006135500 x1855007612436224/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074888 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074872.830253:0:39237:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830355:0:39237:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074872.830364:0:39237:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074872.830394:0:39237:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:3.0:1769074872.830402:0:39237:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:3.0:1769074872.830408:0:39237:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074872.830412:0:39237:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914154bcb8 00010000:00010000:3.0:1769074872.830415:0:39237:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074872.830425:0:39237:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074872.837614:0:39237:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074872.839336:0:39237:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074872.839370:0:39237:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074872.839389:0:39237:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074872.839406:0:39237:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074872.839414:0:39237:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074872.839425:0:39237:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074872.839430:0:39237:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:1.0:1769074872.839438:0:39237:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914154bcb8 00010000:00010000:1.0:1769074872.839441:0:39237:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074872.839449:0:39237:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074872.839634:0:39237:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074872.839680:0:39237:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074872.839687:0:39237:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074872.839696:0:39237:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074872.839702:0:39237:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074872.840054:0:39237:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074872.840059:0:39237:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:1.0:1769074872.840065:0:39237:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914154bbb0 00010000:00010000:1.0:1769074872.840068:0:39237:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074872.840077:0:39237:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074872.840097:0:39237:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074872.840107:0:39237:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:1.0:1769074872.840288:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:1.0:1769074872.840308:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c40c00 uarg=0000000000000000 00800000:00000004:1.0:1769074872.840315:0:39237:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1857, max_age 1865 00000100:00100000:1.0:1769074872.840374:0:39237:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436352:192.168.204.102@tcp:41:lfs.0 00000100:00100000:1.0:1769074872.840470:0:39237:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10030e23c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074872.844942:0:39237:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436352:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074872.844974:0:39237:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501126 00000080:00200080:3.0:1769074872.846668:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846692:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846702:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846873:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846883:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846889:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846893:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846897:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846901:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846904:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846908:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846913:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846917:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846921:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846926:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846931:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846935:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846940:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846944:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846948:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846952:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846955:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846959:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846963:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846968:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846971:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846975:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846979:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846983:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846986:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846990:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.846994:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.846998:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847001:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847005:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847009:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847012:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847016:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847020:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847023:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847027:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847030:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847034:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847037:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847042:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847045:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847049:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847053:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847057:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847061:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847065:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847068:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847072:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847075:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847079:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847092:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847098:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847102:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847106:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847110:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847114:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847118:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847122:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847126:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847130:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847134:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847138:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847142:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847146:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847150:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847154:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847158:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847162:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847165:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847169:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847174:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847177:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847180:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847183:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847186:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847189:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847193:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847196:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847199:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847202:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847206:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847209:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847213:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847216:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847220:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847223:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847227:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847231:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847234:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847238:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847242:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847246:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847249:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847253:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847256:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847260:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847264:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847267:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847270:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847274:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847277:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847281:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847284:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847288:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847291:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847295:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847298:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847302:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847305:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847308:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847311:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847315:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847319:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847323:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847327:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847331:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847335:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847338:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847342:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847345:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847349:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847353:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00800000:00000080:3.0:1769074872.847357:0:39237:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.847361:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.847367:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00020000:00000004:3.0:1769074872.847375:0:39237:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1857, max_age 1865 00000100:00100000:3.0:1769074872.847424:0:39237:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436480:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074872.847511:0:39237:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074872.854904:0:39237:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436480:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074872.854934:0:39237:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074872.855061:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.855080:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00020000:00000004:3.0:1769074872.855099:0:39237:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1857, max_age 1865 00000100:00100000:3.0:1769074872.855133:0:39237:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436608:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074872.855220:0:39237:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10031c3240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074872.859924:0:39237:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39237:1855007612436608:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074872.859956:0:39237:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:3.0:1769074872.862214:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862240:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862250:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862255:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862260:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862369:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862377:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862381:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862386:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862389:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862394:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862397:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862402:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862405:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862409:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862413:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862417:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862421:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862425:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862429:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862433:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862437:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862441:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862444:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862449:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862453:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862457:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862462:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862466:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862469:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862474:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862477:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862482:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862485:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862490:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862494:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862498:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862502:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862506:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862510:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862515:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862518:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862522:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862526:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862530:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862534:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862538:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862541:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862545:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862550:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862553:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862557:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862561:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862564:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862568:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862572:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862575:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862579:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862583:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862587:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862591:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862594:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862598:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862601:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862606:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862609:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862613:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862616:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862620:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862624:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862628:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862631:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862635:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862824:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862833:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862837:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862842:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862845:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862849:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862853:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862858:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862861:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862866:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862870:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862875:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862879:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862883:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862887:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862891:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862895:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862899:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862903:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862908:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862911:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862916:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862920:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862925:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862929:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862933:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862937:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862941:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862945:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862949:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862952:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862956:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862960:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862964:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862967:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862971:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862975:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862979:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862983:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862987:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862990:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.862995:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.862999:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.863003:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.863007:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.863011:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.863015:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.863018:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.863022:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.863026:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.863030:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074872.863034:0:39237:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff2c24d670 00020000:00000080:3.0:1769074872.863038:0:39237:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200008:3.0:1769074872.863054:0:39237:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:3.0:1769074872.870283:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074872.904557:0:39253:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074872.904589:0:39253:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074872.904607:0:39253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.904629:0:39253:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074872.904636:0:39253:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074872.905285:0:39253:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074872.905289:0:39253:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:2.0:1769074872.905297:0:39253:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa91416dfb58 00010000:00010000:2.0:1769074872.905300:0:39253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.905309:0:39253:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074872.905324:0:39253:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074872.905339:0:39253:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074872.905343:0:39253:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074872.905357:0:39253:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f10.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074872.905405:0:39253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.905413:0:39253:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.905421:0:39253:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39253 initiator: MDT0 00000100:00100000:2.0:1769074872.905446:0:39253:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:39253:1855007612436736:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:2.0:1769074872.905590:0:39253:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1003760cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074872.916305:0:39253:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:39253:1855007612436736:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:2.0:1769074872.916327:0:39253:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916342:0:39253:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916348:0:39253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916356:0:39253:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916363:0:39253:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916368:0:39253:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698400 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074872.916373:0:39253:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916380:0:39253:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916386:0:39253:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916393:0:39253:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074872.916400:0:39253:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 39253 initiator: MDT0 00010000:00010000:2.0:1769074872.916406:0:39253:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698400/0x68a7113811d80653 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 39253 initiator: MDT0 00000002:00100000:2.0:1769074872.916420:0:39253:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1005d5b800 x1855007612436736/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074888 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'checkstat.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074872.916434:0:39253:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074872.916459:0:39253:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.916472:0:39253:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074872.916481:0:39253:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa91416df940 00010000:00010000:2.0:1769074872.916484:0:39253:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.916492:0:39253:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074872.923707:0:39255:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074872.923736:0:39255:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074872.923756:0:39255:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.923775:0:39255:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074872.923783:0:39255:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074872.923792:0:39255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074872.923796:0:39255:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:2.0:1769074872.923803:0:39255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa9140d7fb58 00010000:00010000:2.0:1769074872.923807:0:39255:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074872.923822:0:39255:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074872.923843:0:39255:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074872.923858:0:39255:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f10.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074872.923863:0:39255:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f10.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074872.923877:0:39255:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f10.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074872.923923:0:39255:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39255 initiator: MDT0 00010000:00010000:2.0:1769074872.923931:0:39255:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39255 initiator: MDT0 00010000:00010000:2.0:1769074872.923940:0:39255:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39255 initiator: MDT0 00000100:00100000:2.0:1769074872.923966:0:39255:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:39255:1855007612436864:192.168.204.102@tcp:101:checkstat.0 00000100:00100000:2.0:1769074872.924063:0:39255:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1003760cc0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074872.934154:0:39255:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job checkstat:93598d0a-9484-4e40-94b2-393ee542cbad:39255:1855007612436864:192.168.204.102@tcp:101:checkstat.0 00010000:00010000:0.0:1769074872.934181:0:39255:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934203:0:39255:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934212:0:39255:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934223:0:39255:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934234:0:39255:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934246:0:39255:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934253:0:39255:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934261:0:39255:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074872.934273:0:39255:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5b800 x1855007612436864/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074888 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074872.934285:0:39255:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934295:0:39255:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/275 (0 0) 00000080:00200000:0.0:1769074872.934301:0:39255:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074872.934323:0:39255:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x113:0x0], updating i_size 0 00000080:00200000:0.0:1769074872.934327:0:39255:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x113:0x0](ffff8b100485ec08): rc = 0 00000080:00200000:0.0:1769074872.934343:0:39255:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x113:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074872.934350:0:39255:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) for lock 0x68a7113811d8065a 00000080:00010000:0.0:1769074872.934360:0:39255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) 00000080:00010000:0.0:1769074872.934364:0:39255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8065a from it ffffaa9140d7fb20 00010000:00010000:0.0:1769074872.934367:0:39255:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934376:0:39255:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074872.934420:0:39255:0:(file.c:6252:ll_getattr_dentry()) START file f10.replay-single-2:[0x200001b71:0x113:0x0](ffff8b100485ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074872.934426:0:39255:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x113:0x0](ffff8b100485ec08),name=f10.replay-single-2 00000002:00010000:0.0:1769074872.934446:0:39255:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x113:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074872.934454:0:39255:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934465:0:39255:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.934471:0:39255:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074872.934477:0:39255:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) 00000080:00010000:0.0:1769074872.934480:0:39255:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) for lock 0x68a7113811d8065a 00000080:00010000:0.0:1769074872.934483:0:39255:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8065a from it ffffaa9140d7fcb8 00010000:00010000:0.0:1769074872.934486:0:39255:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.934492:0:39255:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942593:0:39257:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074872.942626:0:39257:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074872.942670:0:39257:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942689:0:39257:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942698:0:39257:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074872.942707:0:39257:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074872.942711:0:39257:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:0.0:1769074872.942717:0:39257:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914156fb58 00010000:00010000:0.0:1769074872.942721:0:39257:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942729:0:39257:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942745:0:39257:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074872.942753:0:39257:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single-2, flags=8 00000080:00200008:0.0:1769074872.942762:0:39257:0:(file.c:6252:ll_getattr_dentry()) START file f10.replay-single-2:[0x200001b71:0x113:0x0](ffff8b100485ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074872.942766:0:39257:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x113:0x0](ffff8b100485ec08),name=f10.replay-single-2 00000002:00010000:0.0:1769074872.942778:0:39257:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x113:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074872.942785:0:39257:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942793:0:39257:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942800:0:39257:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074872.942805:0:39257:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) 00000080:00010000:0.0:1769074872.942808:0:39257:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffff8b100485ec08) for lock 0x68a7113811d8065a 00000080:00010000:0.0:1769074872.942812:0:39257:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8065a from it ffffaa914156fcb8 00010000:00010000:0.0:1769074872.942815:0:39257:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942822:0:39257:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942855:0:39257:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074872.942861:0:39257:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074872.942867:0:39257:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942875:0:39257:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942881:0:39257:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074872.942884:0:39257:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074872.942887:0:39257:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:0.0:1769074872.942890:0:39257:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914156fc10 00010000:00010000:0.0:1769074872.942893:0:39257:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942899:0:39257:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942906:0:39257:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074872.942912:0:39257:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single-2, flags=0 00000080:00200000:0.0:1769074872.942916:0:39257:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074872.942922:0:39257:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074872.942927:0:39257:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942934:0:39257:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942940:0:39257:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074872.942943:0:39257:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074872.942945:0:39257:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8064c 00000080:00010000:0.0:1769074872.942948:0:39257:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8064c from it ffffaa914156fd68 00010000:00010000:0.0:1769074872.942951:0:39257:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.942957:0:39257:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.942964:0:39257:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:0.0:1769074872.942969:0:39257:0:(namei.c:2354:ll_unlink()) VFS Op:name=f10.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:0.0:1769074872.942998:0:39257:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.943030:0:39257:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074872.943041:0:39257:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x113:0x0]: losing layout lock 00010000:00010000:0.0:1769074872.946485:0:39257:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.946513:0:39257:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.946519:0:39257:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074872.946522:0:39257:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699800/0x68a7113811d8065a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf30647f5 expref: -99 pid: 39255 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074872.946544:0:39257:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8400/0x68a7113811d8064c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30647c4 expref: -99 pid: 39237 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074872.946573:0:39257:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074872.946587:0:39257:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006026680 x1855007612437120/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074872.946615:0:39257:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39257:1855007612437120:192.168.204.102@tcp:36:rm.0 00000100:00100000:0.0:1769074872.949151:0:39257:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088659c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074872.968149:0:39257:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39257:1855007612437120:192.168.204.102@tcp:36:rm.0 00000080:00200000:2.0:1769074872.968215:0:39257:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x113:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074872.968226:0:39257:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x113:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074872.968234:0:39257:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x113:0x0](ffff8b100485ec08) 00010000:00010000:2.0:1769074873.896930:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074874.016255:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612437248:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074874.016380:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437504:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074874.016415:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074874.016421:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437376:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074874.016444:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437632:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074874.038178:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437376:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074874.038195:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612437248:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074874.047296:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437632:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074874.047316:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612437504:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074874.911848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074875.938114:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074876.963958:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074877.983902:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074879.007850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074879.152487:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612437760:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074879.152581:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612438016:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074879.152615:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074879.152621:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612437888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074879.152670:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612438144:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074879.164759:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612437760:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074879.164782:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612438016:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074879.172802:0:2364:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280513 gen 1 00000100:00100000:0.0:1769074879.172837:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612437888:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074879.172855:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612438144:192.168.204.102@tcp:400:kworker.0 00000001:02000400:0.0:1769074879.243211:0:39822:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 11: create open write rename |X| create-old-name read ========================================================== 04:41:18 (1769074878) 00000080:00200000:3.0:1769074879.582847:0:39834:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074879.582876:0:39834:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074879.582941:0:39834:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39834 initiator: MDT0 00010000:00010000:3.0:1769074879.582952:0:39834:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39834 initiator: MDT0 00010000:00010000:3.0:1769074879.582962:0:39834:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39834 initiator: MDT0 00000100:00100000:3.0:1769074879.582996:0:39834:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39834:1855007612438272:192.168.204.102@tcp:101:mcreate.0 00000100:00100000:3.0:1769074879.583083:0:39834:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364ccc0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074879.597740:0:39834:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39834:1855007612438272:192.168.204.102@tcp:101:mcreate.0 00010000:00010000:1.0:1769074879.597770:0:39834:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597783:0:39834:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597796:0:39834:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597808:0:39834:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597820:0:39834:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597828:0:39834:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597835:0:39834:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074879.597851:0:39834:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de300 x1855007612438272/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074879.597864:0:39834:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597878:0:39834:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1769074879.597883:0:39834:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074879.597913:0:39834:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:1.0:1769074879.597919:0:39834:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80661 00000080:00010000:1.0:1769074879.597924:0:39834:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074879.597927:0:39834:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80661 from it ffffaa914154bbd0 00010000:00010000:1.0:1769074879.597930:0:39834:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.597939:0:39834:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074879.597964:0:39834:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074879.597999:0:39834:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:1.0:1769074879.598004:0:39834:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074879.598018:0:39834:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074879.598026:0:39834:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598036:0:39834:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074879.598047:0:39834:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074879.598050:0:39834:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074879.598053:0:39834:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80661 00000080:00010000:1.0:1769074879.598057:0:39834:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80661 from it ffffaa914154bcd0 00010000:00010000:1.0:1769074879.598060:0:39834:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598066:0:39834:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074879.598073:0:39834:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074879.598079:0:39834:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074879.598085:0:39834:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074879.598089:0:39834:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598097:0:39834:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074879.598102:0:39834:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074879.598105:0:39834:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074879.598108:0:39834:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80661 00000080:00010000:1.0:1769074879.598111:0:39834:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80661 from it ffffaa914154bdb0 00010000:00010000:1.0:1769074879.598114:0:39834:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598120:0:39834:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074879.598128:0:39834:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074879.598133:0:39834:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:1.0:1769074879.598137:0:39834:0:(namei.c:2020:ll_mknod()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:1.0:1769074879.598159:0:39834:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598198:0:39834:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074879.598205:0:39834:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074879.598207:0:39834:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169c00/0x68a7113811d80661 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306481f expref: -99 pid: 39834 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074879.598233:0:39834:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074879.598239:0:39834:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006134a80 x1855007612438528/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074879.598260:0:39834:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39834:1855007612438528:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:1.0:1769074879.598340:0:39834:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53180 going to sleep for 11 seconds 00000100:00100000:3.0:1769074879.634377:0:39834:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39834:1855007612438528:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:3.0:1769074879.634441:0:39834:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00200000:3.0:1769074879.634483:0:39834:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f11.replay-single, unhashed 1 00000080:00200000:2.0:1769074879.635592:0:39691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.635852:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.635913:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.635923:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.635934:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000100:00100000:2.0:1769074879.635959:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612438656:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074879.636061:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.656341:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612438656:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074879.656371:0:39691:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656384:0:39691:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656397:0:39691:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656420:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656436:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656444:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656450:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074879.656461:0:39691:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5bb80 x1855007612438656/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074879.656474:0:39691:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656487:0:39691:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074879.656494:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074879.656522:0:39691:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074879.656541:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80668 00000080:00010000:2.0:1769074879.656547:0:39691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.656550:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80668 from it ffffaa91416dfbf8 00010000:00010000:2.0:1769074879.656554:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656562:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.656590:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.656629:0:39691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.656675:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.656685:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656696:0:39691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.656702:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.656707:0:39691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.656710:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80668 00000080:00010000:2.0:1769074879.656714:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80668 from it ffffaa91416dfb38 00010000:00010000:2.0:1769074879.656717:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656724:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.656731:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074879.656736:0:39691:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1008384d80, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1769074879.656745:0:39691:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open|creat 00000002:00010000:2.0:1769074879.656763:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single,[0x200001b71:0x116:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1769074879.656775:0:39691:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656820:0:39691:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656827:0:39691:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074879.656831:0:39691:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80668 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306483b expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.656867:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.656875:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000020:00100000:2.0:1769074879.656886:0:39691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074879.656893:0:39691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5bb80 x1855007612438912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074879.656901:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000100:00100000:2.0:1769074879.656922:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612438912:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074879.657099:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.662737:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612438912:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074879.662760:0:39691:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662772:0:39691:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662778:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662785:0:39691:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 3/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662792:0:39691:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662798:0:39691:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005698a00 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074879.662802:0:39691:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 2 type: IBT flags: 0x206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662808:0:39691:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 4/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 2 type: IBT flags: 0x206400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662814:0:39691:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 5/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 2 type: IBT flags: 0x20e400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662819:0:39691:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074879.662825:0:39691:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 3/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.662830:0:39691:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8066f lrc: 0/0,0 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 39691 initiator: MDT0 00000002:00100000:2.0:1769074879.662853:0:39691:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=800002b, status=0 req@ffff8b1005d5bb80 x1855007612438912/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074895 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074879.662866:0:39691:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5bb80 x1855007612438912/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074895 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074879.662882:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074879.662923:0:39691:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074879.662933:0:39691:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074879.662985:0:39691:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x115:0x0], updating i_size 0 00000080:00200000:2.0:1769074879.662988:0:39691:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffff8b1004859148): rc = 0 00000080:00010000:2.0:1769074879.662993:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x0 00000080:00200008:2.0:1769074879.663012:0:39691:0:(file.c:1001:ll_file_open()) START file f11.replay-single:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 101101 00000002:00100000:2.0:1769074879.663031:0:39691:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5bb80 x1855007612438912/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074895 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074879.663075:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148), inode mode 81a4 mask 42 00000080:00200008:2.0:1769074879.663088:0:39691:0:(llite_lib.c:2314:ll_setattr_raw()) START file f11.replay-single:[0x200001b71:0x115:0x0](ffff8b1004859148) current size 0, valid attrs a068, mode 0, uid 2, gid 0, new size 0, atime 1769074879.000000000, mtime 1769074879.000000000, ctime 1769074879.000000000 00000020:00100000:2.0:1769074879.663139:0:39691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074879.663146:0:39691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5b800 x1855007612439168/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074879.663166:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439168:192.168.204.102@tcp:36:bash.0 00000100:00100000:2.0:1769074879.663234:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.667036:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439168:192.168.204.102@tcp:36:bash.0 00000080:00200000:2.0:1769074879.667093:0:39691:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000080:00200000:2.0:1769074879.667101:0:39691:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:2.0:1769074879.667121:0:39691:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200001b71:0x115:0x0](ffff8b1004859148) 00000002:00010000:2.0:1769074879.667131:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x115:0x0]) in obj [0x200001b71:0x115:0x0], intent: layout flags 0 00010000:00010000:2.0:1769074879.667177:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.667187:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074879.667196:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000100:00100000:2.0:1769074879.667215:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439296:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074879.675276:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.678988:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439296:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074879.679019:0:39691:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679036:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074879.679049:0:39691:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b1005d58700 x1855007612439296/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 376/400 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074879.679060:0:39691:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074879.679072:0:39691:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679086:0:39691:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 16 res 8589941617/277 (0 0) 00000080:00200000:2.0:1769074879.679092:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200001b71:0x115:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074879.679111:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d80676 00000080:00010000:2.0:1769074879.679119:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80676 from it ffffaa91416df8c8 00010000:00010000:2.0:1769074879.679123:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679133:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679153:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 2/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679163:0:39691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 2/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074879.679171:0:39691:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200001b71:0x115:0x0](00000000b5930f5a) being reconfigured ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074879.679209:0:39691:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 4294967294 -> 0 00010000:00010000:2.0:1769074879.679214:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.679221:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00020000:00200000:2.0:1769074879.679243:0:39691:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074879.679258:0:39691:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1769074879.679270:0:39691:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 4 00000008:00010000:2.0:1769074879.679275:0:39691:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef4b0, osc lock ffff8b1010ac7a20, flags 0x80000000 00010000:00010000:2.0:1769074879.679318:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.679327:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x80000000 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.679336:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074879.679357:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439424:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074879.679436:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.685476:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439424:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074879.685509:0:39691:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.685524:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.685550:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074879.685563:0:39691:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.685576:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074879.685583:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074879.685741:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612439552:192.168.204.102@tcp:10:bash.0 00000100:00100000:2.0:1769074879.694874:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612439552:192.168.204.102@tcp:10:bash.0 00000080:00200000:0.0:1769074879.695029:0:39691:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 0 i_blocks 1 00000020:00010000:0.0:1769074879.695037:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074879.695041:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074879.695045:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef4b0 (ffff8b1007c69e18/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074879.695056:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695069:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074879.695085:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074879.695088:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef4b0 (ffff8b1007c696c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074879.695109:0:39691:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074879.695124:0:39691:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f11.replay-single:[0x200001b71:0x115:0x0](ffff8b1004859148) current size 0, valid attrs a168, mode 0, uid 2, gid 0, new size 0, atime 1769074879.000000000, mtime 1769074879.000000000, ctime 1769074879.000000000, rc 0 00000080:00200008:0.0:1769074879.695218:0:39691:0:(file.c:2648:do_file_write_iter()) START file f11.replay-single:[0x200001b71:0x115:0x0], ppos: 0, count: 4 00000080:00200000:0.0:1769074879.695230:0:39691:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:0.0:1769074879.695250:0:39691:0:(file.c:2624:ll_do_tiny_write()) result: 0, original count 4 00000080:00200000:0.0:1769074879.695255:0:39691:0:(file.c:1994:ll_file_io_generic()) f11.replay-single: write ppos: 0, bytes: 4 00000080:00200000:0.0:1769074879.695260:0:39691:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:0.0:1769074879.695274:0:39691:0:(cl_io.c:214:cl_io_rw_init()) header@000000007469e5a4[0x4, 1, [0x200001b71:0x115:0x0] hash] 00000020:00200000:0.0:1769074879.695277:0:39691:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4) 0 0 00000080:00200000:0.0:1769074879.695281:0:39691:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074879.695293:0:39691:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:0.0:1769074879.695300:0:39691:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4) 4, 4 00020000:00200000:0.0:1769074879.695304:0:39691:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074879.695318:0:39691:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:0.0:1769074879.695334:0:39691:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074879.695338:0:39691:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972da30, osc lock ffff8b10097a2f70, flags 0x0 00010000:00010000:0.0:1769074879.695356:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695369:0:39691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695380:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695388:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/0,2 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695394:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074879.695405:0:39691:0:(vvp_io.c:1312:vvp_io_write_start()) f11.replay-single: write [0, 4) append 0 append lockpos 0 00000080:00200000:0.0:1769074879.695426:0:39691:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:0.0:1769074879.695469:0:39691:0:(rw26.c:912:ll_write_end()) pos 0, len 4, copied 4 00000080:00200000:0.0:1769074879.695477:0:39691:0:(rw26.c:969:ll_write_end()) page@00000000be31049b[3 00000000c101dd59 2 1 000000000a3807c2] 00000080:00200000:0.0:1769074879.695480:0:39691:0:(rw26.c:969:ll_write_end()) vmpage @0000000009632d24 17ffffc0001001 3:0 ffff8b100572b5e8 0 lru 00000080:00200000:0.0:1769074879.695487:0:39691:0:(rw26.c:969:ll_write_end()) osc-page@00000000dd84c059 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000005877d1e7 00000000b64bb2a9 > 3< 0 0 > 4< 0 0 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074879.695490:0:39691:0:(rw26.c:969:ll_write_end()) end page@00000000be31049b 00000080:00200000:0.0:1769074879.695491:0:39691:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:0.0:1769074879.695497:0:39691:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 1, from 0, to 4 prio 0 00000008:00010000:0.0:1769074879.695516:0:39691:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000607cb37b ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074879.695545:0:39691:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff8b10048592f8, count 1, dirtied 1 00000080:00200000:0.0:1769074879.695549:0:39691:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 1 pages 4 bytes, tot: 4 00000080:00200000:0.0:1769074879.695552:0:39691:0:(vvp_io.c:1390:vvp_io_write_start()) f11.replay-single: write bytes 0, result: 4 00000020:00010000:0.0:1769074879.695569:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074879.695571:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074879.695574:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972da30 (ffff8b1007c69e18/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074879.695577:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074879.695585:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074879.695601:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1006124d58 (ffff8b1007c696c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074879.695604:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972da30 (ffff8b1007c696c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074879.695610:0:39691:0:(vvp_io.c:518:vvp_io_advance()) advancing 4 bytes 00000080:00200000:0.0:1769074879.695614:0:39691:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:0.0:1769074879.695619:0:39691:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074879.695622:0:39691:0:(file.c:2142:ll_file_io_generic()) f11.replay-single: 2 io complete with rc: 0, result: 4, restart: 0 00000080:00200000:0.0:1769074879.695627:0:39691:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 1 00000080:00200000:0.0:1769074879.695630:0:39691:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 4 00000080:00200008:0.0:1769074879.695691:0:39691:0:(file.c:424:ll_file_release()) START file f11.replay-single:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 100001 00000002:00100000:0.0:1769074879.695750:0:39691:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005d5bb80 x1855007612438912/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074895 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074879.695761:0:39691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074879.695766:0:39691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006026a00 x1855007612439808/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074879.695792:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439808:192.168.204.102@tcp:35:bash.0 00000100:00100000:0.0:1769074879.695863:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec9c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074879.708473:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612439808:192.168.204.102@tcp:35:bash.0 00000002:00100000:0.0:1769074879.708510:0:39691:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1005d5bb80 x1855007612438912/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074879.730569:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074879.730769:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074879.730837:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39835 initiator: MDT0 00010000:00010000:1.0:1769074879.730849:0:39835:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00010000:00010000:1.0:1769074879.730860:0:39835:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00000100:00100000:1.0:1769074879.730892:0:39835:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612439936:192.168.204.102@tcp:101:mv.0 00000100:00100000:1.0:1769074879.731041:0:39835:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53780 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.733758:0:39835:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612439936:192.168.204.102@tcp:101:mv.0 00010000:00010000:2.0:1769074879.733783:0:39835:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733795:0:39835:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733805:0:39835:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733816:0:39835:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733830:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733838:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733845:0:39835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074879.733855:0:39835:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134700 x1855007612439936/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074879.733867:0:39835:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733879:0:39835:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074879.733885:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074879.733911:0:39835:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074879.733917:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.733922:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.733926:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bbd0 00010000:00010000:2.0:1769074879.733929:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.733937:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.733958:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.734010:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.734035:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.734043:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.734053:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.734060:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.734064:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.734067:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.734071:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bbd0 00010000:00010000:2.0:1769074879.734074:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.734081:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.734089:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.734105:0:39835:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=0 00000080:00200000:2.0:1769074879.734109:0:39835:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074879.734122:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.734158:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.734165:0:39835:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.734173:0:39835:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00000100:00100000:2.0:1769074879.734197:0:39835:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440064:192.168.204.102@tcp:101:mv.0 00000100:00100000:2.0:1769074879.734607:0:39835:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.755634:0:39835:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440064:192.168.204.102@tcp:101:mv.0 00010000:00010000:2.0:1769074879.755707:0:39835:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755724:0:39835:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755732:0:39835:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755744:0:39835:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755756:0:39835:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755767:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755774:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.755781:0:39835:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074879.755791:0:39835:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d58a80 x1855007612440064/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/640 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074879.755803:0:39835:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074879.755816:0:39835:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.755826:0:39835:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:2.0:1769074879.755832:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074879.755878:0:39835:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffff8b1004859148): rc = 0 00000080:00200000:2.0:1769074879.755893:0:39835:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1769074879.755899:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d8068b 00000080:00010000:2.0:1769074879.755910:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00010000:2.0:1769074879.755914:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8068b from it ffffaa914164bdb8 00010000:00010000:2.0:1769074879.755918:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.755926:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074879.755965:0:39835:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=2048 00000080:00200000:2.0:1769074879.755980:0:39835:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074879.756000:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.756040:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.756048:0:39835:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.756056:0:39835:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00000100:00100000:2.0:1769074879.756076:0:39835:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440192:192.168.204.102@tcp:101:mv.0 00000100:00100000:2.0:1769074879.756138:0:39835:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.760676:0:39835:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440192:192.168.204.102@tcp:101:mv.0 00010000:00010000:2.0:1769074879.760700:0:39835:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760713:0:39835:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760720:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760727:0:39835:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760734:0:39835:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760740:0:39835:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff8b1005699400 already unused, calling callback (ffffffffc19e3c80) 00010000:00010000:2.0:1769074879.760745:0:39835:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760752:0:39835:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760759:0:39835:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760766:0:39835:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074879.760773:0:39835:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 39835 initiator: MDT0 00010000:00010000:2.0:1769074879.760783:0:39835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d80692 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 39835 initiator: MDT0 00000002:00100000:2.0:1769074879.760802:0:39835:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff8b1005d58a80 x1855007612440192/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074895 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'mv.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074879.760816:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074879.760841:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.760854:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074879.760864:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bbd8 00010000:00010000:2.0:1769074879.760868:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.760876:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.760910:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.760923:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.760930:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.760938:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.760945:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.760950:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.760954:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761090:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bc98 00010000:00010000:2.0:1769074879.761095:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761356:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761368:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074879.761375:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761383:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761391:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761398:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761403:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761407:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761409:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761412:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bcc8 00010000:00010000:2.0:1769074879.761415:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761422:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761429:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074879.761470:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761477:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761482:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761490:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761496:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761499:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761502:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761505:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bb58 00010000:00010000:2.0:1769074879.761508:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761515:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761523:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.761529:0:39835:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single-2, flags=8 00000080:00200000:2.0:1769074879.761542:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761548:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761554:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761561:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761567:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761570:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761573:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761576:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bbd0 00010000:00010000:2.0:1769074879.761579:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761585:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761591:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.761598:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761604:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761609:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761617:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761622:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761626:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761628:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761631:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bbd0 00010000:00010000:2.0:1769074879.761633:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761686:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761694:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074879.761701:0:39835:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single, flags=0 00000080:00200000:2.0:1769074879.761704:0:39835:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single-2, flags=2048 00000080:00200000:2.0:1769074879.761706:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761713:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761718:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761726:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761732:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761736:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761738:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761741:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bc98 00010000:00010000:2.0:1769074879.761744:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761751:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761758:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074879.761761:0:39835:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074879.761767:0:39835:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074879.761772:0:39835:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761779:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761785:0:39835:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074879.761789:0:39835:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074879.761792:0:39835:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80684 00000080:00010000:2.0:1769074879.761796:0:39835:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80684 from it ffffaa914164bcc8 00010000:00010000:2.0:1769074879.761799:0:39835:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761805:0:39835:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074879.761813:0:39835:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074879.761817:0:39835:0:(namei.c:2433:ll_rename()) VFS Op:oldname=f11.replay-single, src_dir=[0x200000007:0x1:0x0](ffff8b1018194b08), newname=f11.replay-single-2, tgt_dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:2.0:1769074879.761839:0:39835:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761861:0:39835:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.761871:0:39835:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.761908:0:39835:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.761915:0:39835:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074879.761921:0:39835:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074879.761923:0:39835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d8068b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf306489d expref: -99 pid: 39835 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074879.761934:0:39835:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8c00/0x68a7113811d80684 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064881 expref: -99 pid: 39835 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074879.761960:0:39835:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074879.761980:0:39835:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58a80 x1855007612440448/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:13/10 lens 536/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074879.762003:0:39835:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440448:192.168.204.102@tcp:36:mv.0 00000100:00100000:2.0:1769074879.762602:0:39835:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074879.785928:0:39835:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job mv:93598d0a-9484-4e40-94b2-393ee542cbad:39835:1855007612440448:192.168.204.102@tcp:36:mv.0 00010000:00010000:0.0:1769074880.031839:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074880.923272:0:39844:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074880.923292:0:39844:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074880.923319:0:39844:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074880.923410:0:39844:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39844 initiator: MDT0 00010000:00010000:0.0:1769074880.923421:0:39844:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39844 initiator: MDT0 00010000:00010000:0.0:1769074880.923433:0:39844:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39844 initiator: MDT0 00000100:00100000:0.0:1769074880.923464:0:39844:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440576:192.168.204.102@tcp:101:lfs.0 00000100:00100000:0.0:1769074880.924213:0:39844:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074880.937214:0:39844:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440576:192.168.204.102@tcp:101:lfs.0 00010000:00010000:3.0:1769074880.937239:0:39844:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937252:0:39844:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937264:0:39844:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937274:0:39844:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937285:0:39844:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937293:0:39844:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937300:0:39844:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074880.937310:0:39844:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006025180 x1855007612440576/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074896 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074880.937322:0:39844:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937338:0:39844:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074880.937345:0:39844:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074880.937373:0:39844:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:3.0:1769074880.937378:0:39844:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:3.0:1769074880.937383:0:39844:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074880.937386:0:39844:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa914176bcb8 00010000:00010000:3.0:1769074880.937389:0:39844:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.937397:0:39844:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074880.938519:0:39844:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074880.938527:0:39844:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074880.938541:0:39844:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074880.938552:0:39844:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.938565:0:39844:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074880.938583:0:39844:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074880.938588:0:39844:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074880.938592:0:39844:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:3.0:1769074880.938597:0:39844:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa914176bcb8 00010000:00010000:3.0:1769074880.938600:0:39844:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.938613:0:39844:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074880.938739:0:39844:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074880.938746:0:39844:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074880.938752:0:39844:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.938760:0:39844:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074880.938767:0:39844:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074880.938770:0:39844:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074880.938773:0:39844:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:3.0:1769074880.938776:0:39844:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa914176bbb0 00010000:00010000:3.0:1769074880.938779:0:39844:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074880.938786:0:39844:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074880.938793:0:39844:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:3.0:1769074880.938799:0:39844:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:3.0:1769074880.938881:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.938894:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00800000:00000004:3.0:1769074880.938901:0:39844:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1866, max_age 1873 00000100:00100000:3.0:1769074880.938936:0:39844:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440704:192.168.204.102@tcp:41:lfs.0 00000100:00100000:3.0:1769074880.939027:0:39844:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074880.954986:0:39844:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440704:192.168.204.102@tcp:41:lfs.0 00800000:00000004:3.0:1769074880.955014:0:39844:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500778/501125 00000080:00200080:3.0:1769074880.955098:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955117:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955125:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955129:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955133:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955137:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955140:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955144:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955147:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955151:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955155:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955158:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955162:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955166:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955169:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955172:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955175:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955178:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955182:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955185:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955188:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955192:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955195:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955197:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955200:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955203:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955206:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955209:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955213:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955215:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955218:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955221:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955224:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955227:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955230:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955232:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955235:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955238:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955240:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955243:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955247:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955250:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955253:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955255:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955259:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955262:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955266:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955269:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955272:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955275:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955279:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955282:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955285:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955289:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955292:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955295:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955299:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955302:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955306:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955310:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955315:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955318:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955321:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955325:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955328:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955331:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955334:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955338:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955341:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955344:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955348:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955352:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955355:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955358:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955361:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955364:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955368:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955371:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955374:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955378:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955381:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955385:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955388:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955392:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955395:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955399:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955402:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955406:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955409:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955413:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955417:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955420:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955425:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955429:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955433:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955436:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955441:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955444:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955448:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955451:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955455:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955458:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955462:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955465:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955468:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955471:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955475:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955478:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955482:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955485:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955489:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955492:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955496:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955499:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955503:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955507:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955510:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955514:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955518:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955521:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955525:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955529:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955532:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955536:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955540:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955544:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955547:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00800000:00000080:3.0:1769074880.955551:0:39844:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00000080:00200080:3.0:1769074880.955555:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:3.0:1769074880.955561:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f37c00 uarg=0000000000000000 00020000:00000004:3.0:1769074880.955567:0:39844:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1866, max_age 1873 00000100:00100000:3.0:1769074880.955604:0:39844:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440832:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074880.955708:0:39844:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074880.990202:0:39844:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440832:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074880.990232:0:39844:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074880.990418:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.990437:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00020000:00000004:1.0:1769074880.990445:0:39844:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1866, max_age 1873 00000100:00100000:1.0:1769074880.990483:0:39844:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440960:192.168.204.102@tcp:13:lfs.0 00000100:00100000:1.0:1769074880.991081:0:39844:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53780 going to sleep for 11 seconds 00000100:00100000:1.0:1769074880.998558:0:39844:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:39844:1855007612440960:192.168.204.102@tcp:13:lfs.0 00020000:00000004:1.0:1769074880.998587:0:39844:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118190 00000080:00200080:1.0:1769074880.998696:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998713:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998720:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998724:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998729:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998733:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998737:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998740:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998745:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998748:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998751:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998754:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998758:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998763:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998767:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998769:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998774:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998777:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998781:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998784:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998787:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998790:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998794:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998797:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998802:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998805:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998810:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998813:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998816:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998820:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998824:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998827:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998831:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998835:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998839:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998842:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998846:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998849:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998853:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998858:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998861:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998864:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998867:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998870:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998874:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998877:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998880:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998883:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998886:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998888:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998891:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998894:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998898:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998901:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998904:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998907:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998910:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998914:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998917:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998920:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998923:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998926:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998930:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998933:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998938:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998942:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998960:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998965:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998969:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998973:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998977:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998981:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998985:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998988:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998992:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.998995:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.998999:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999003:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999006:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999009:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999013:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999016:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999020:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999024:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999027:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999030:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999034:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999036:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999040:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999043:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999046:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999050:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999054:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999057:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999061:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999064:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999068:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999071:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999075:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999079:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999083:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999086:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999090:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999093:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999096:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999099:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999102:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999105:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999109:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999112:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999115:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999119:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999123:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999126:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999130:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999134:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999137:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999140:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999144:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999147:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999150:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999153:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999157:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999160:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200080:1.0:1769074880.999164:0:39844:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffefd82ace0 00020000:00000080:1.0:1769074880.999167:0:39844:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c42800 uarg=0000000000000000 00000080:00200008:1.0:1769074880.999181:0:39844:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074881.053504:0:39851:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074881.053537:0:39851:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.053557:0:39851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.053572:0:39851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.053580:0:39851:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.053588:0:39851:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074881.053591:0:39851:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:2.0:1769074881.053598:0:39851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa9141bb7bd0 00010000:00010000:2.0:1769074881.053602:0:39851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.053609:0:39851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.053625:0:39851:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.053674:0:39851:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074881.053679:0:39851:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074881.053686:0:39851:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.053691:0:39851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.053699:0:39851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.053705:0:39851:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.053708:0:39851:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074881.053710:0:39851:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:2.0:1769074881.053714:0:39851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa9141bb7cd0 00010000:00010000:2.0:1769074881.053716:0:39851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.053723:0:39851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.053730:0:39851:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074881.053735:0:39851:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074881.053740:0:39851:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.053744:0:39851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.055156:0:39851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.055167:0:39851:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.055176:0:39851:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074881.055180:0:39851:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80699 00000080:00010000:2.0:1769074881.055186:0:39851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80699 from it ffffaa9141bb7db0 00010000:00010000:2.0:1769074881.055189:0:39851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.055196:0:39851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.055210:0:39851:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074881.055216:0:39851:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074881.055220:0:39851:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074881.055252:0:39851:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.055303:0:39851:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.055309:0:39851:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074881.055312:0:39851:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ad000/0x68a7113811d80699 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30648dc expref: -99 pid: 39844 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074881.055341:0:39851:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074881.055352:0:39851:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59180 x1855007612441216/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074881.055375:0:39851:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39851:1855007612441216:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074881.055465:0:39851:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd780 going to sleep for 11 seconds 00010000:00010000:3.0:1769074881.056837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074881.075345:0:39851:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:39851:1855007612441216:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074881.075395:0:39851:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x117:0x0](ffff8b1004868908) 00000080:00200000:2.0:1769074881.075435:0:39851:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:1.0:1769074881.091671:0:39854:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074881.091699:0:39854:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.091762:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39854 initiator: MDT0 00010000:00010000:1.0:1769074881.091773:0:39854:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39854 initiator: MDT0 00010000:00010000:1.0:1769074881.091784:0:39854:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39854 initiator: MDT0 00000100:00100000:1.0:1769074881.091808:0:39854:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441344:192.168.204.102@tcp:101:rm.0 00000100:00100000:1.0:1769074881.092541:0:39854:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074881.099783:0:39854:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441344:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074881.099810:0:39854:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099823:0:39854:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099834:0:39854:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099844:0:39854:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099856:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099864:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099871:0:39854:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074881.099882:0:39854:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134e00 x1855007612441344/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074897 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074881.099897:0:39854:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074881.099909:0:39854:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074881.099916:0:39854:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074881.100194:0:39854:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 17920 00000080:00010000:2.0:1769074881.100203:0:39854:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806a0 00000080:00010000:2.0:1769074881.100209:0:39854:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074881.100213:0:39854:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806a0 from it ffffaa9141c33b58 00010000:00010000:2.0:1769074881.100216:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.100225:0:39854:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.100248:0:39854:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.100266:0:39854:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074881.100271:0:39854:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074881.100292:0:39854:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.100331:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39854 initiator: MDT0 00010000:00010000:2.0:1769074881.100339:0:39854:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39854 initiator: MDT0 00010000:00010000:2.0:1769074881.100348:0:39854:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39854 initiator: MDT0 00000100:00100000:2.0:1769074881.100367:0:39854:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441472:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074881.100540:0:39854:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd780 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.111519:0:39854:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441472:192.168.204.102@tcp:101:rm.0 00010000:00010000:1.0:1769074881.111547:0:39854:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111569:0:39854:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111577:0:39854:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111588:0:39854:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111599:0:39854:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111611:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111618:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.111625:0:39854:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074881.111635:0:39854:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5a680 x1855007612441472/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074897 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074881.113152:0:39854:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113167:0:39854:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/279 (0 0) 00000080:00200000:1.0:1769074881.113173:0:39854:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074881.113193:0:39854:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x117:0x0], updating i_size 0 00000080:00200000:1.0:1769074881.113196:0:39854:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x117:0x0](ffff8b1004868908): rc = 0 00000080:00200000:1.0:1769074881.113214:0:39854:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x117:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1769074881.113220:0:39854:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x117:0x0](ffff8b1004868908) for lock 0x68a7113811d806a7 00000080:00010000:1.0:1769074881.113230:0:39854:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x117:0x0](ffff8b1004868908) 00000080:00010000:1.0:1769074881.113235:0:39854:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806a7 from it ffffaa9141c33b20 00010000:00010000:1.0:1769074881.113238:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113246:0:39854:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074881.113287:0:39854:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x117:0x0](ffff8b1004868908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074881.113293:0:39854:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x117:0x0](ffff8b1004868908),name=fsa-oleg402-client.virtnet 00000002:00010000:1.0:1769074881.113315:0:39854:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x117:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.113323:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113332:0:39854:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113338:0:39854:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074881.113342:0:39854:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x117:0x0](ffff8b1004868908) 00000080:00010000:1.0:1769074881.113345:0:39854:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x117:0x0](ffff8b1004868908) for lock 0x68a7113811d806a7 00000080:00010000:1.0:1769074881.113349:0:39854:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806a7 from it ffffaa9141c33cb8 00010000:00010000:1.0:1769074881.113351:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113358:0:39854:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113424:0:39854:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074881.113432:0:39854:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.113439:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113446:0:39854:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113452:0:39854:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074881.113456:0:39854:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074881.113458:0:39854:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806a0 00000080:00010000:1.0:1769074881.113462:0:39854:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806a0 from it ffffaa9141c33c10 00010000:00010000:1.0:1769074881.113464:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113471:0:39854:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113479:0:39854:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074881.113489:0:39854:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:1.0:1769074881.113493:0:39854:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074881.113499:0:39854:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.113504:0:39854:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113512:0:39854:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113522:0:39854:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074881.113525:0:39854:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074881.113527:0:39854:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806a0 00000080:00010000:1.0:1769074881.113530:0:39854:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806a0 from it ffffaa9141c33d68 00010000:00010000:1.0:1769074881.113533:0:39854:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113539:0:39854:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113546:0:39854:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074881.113551:0:39854:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:1.0:1769074881.113571:0:39854:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113594:0:39854:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.113604:0:39854:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x117:0x0]: losing layout lock 00010000:00010000:1.0:1769074881.113659:0:39854:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113667:0:39854:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113672:0:39854:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074881.113675:0:39854:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699600/0x68a7113811d806a7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064914 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.113697:0:39854:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9000/0x68a7113811d806a0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf30648f8 expref: -99 pid: 39854 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074881.113715:0:39854:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074881.113725:0:39854:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006137480 x1855007612441728/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074881.113749:0:39854:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441728:192.168.204.102@tcp:36:rm.0 00000100:00100000:1.0:1769074881.113835:0:39854:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53180 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.119017:0:39854:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:39854:1855007612441728:192.168.204.102@tcp:36:rm.0 00000080:00200000:1.0:1769074881.119079:0:39854:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x117:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074881.119090:0:39854:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x117:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074881.119098:0:39854:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x117:0x0](ffff8b1004868908) 00010000:00010000:3.0:1769074882.080311:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074883.105078:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074884.133066:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074884.257249:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612441984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.257786:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612442240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.257844:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1769074884.257852:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612441856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.257905:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612442112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.270233:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612442112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.270262:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612441984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.284062:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612441856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074884.284100:0:2366:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967568 gen 10 00000100:00100000:2.0:1769074884.284125:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612442240:192.168.204.102@tcp:400:kworker.0 00000001:02000400:0.0:1769074884.373859:0:39884:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1769074884.396292:0:39691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074884.396331:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074884.396418:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074884.396429:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074884.396441:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000100:00100000:2.0:1769074884.396474:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442368:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074884.396862:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.410448:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442368:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074884.410487:0:39691:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410510:0:39691:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410525:0:39691:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410539:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410553:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410562:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410570:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074884.410582:0:39691:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5aa00 x1855007612442368/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074884.410598:0:39691:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410614:0:39691:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074884.410622:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074884.410743:0:39691:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074884.410752:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806ae 00000080:00010000:2.0:1769074884.410758:0:39691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074884.410763:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806ae from it ffffaa91416dfbf8 00010000:00010000:2.0:1769074884.410767:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410776:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074884.410807:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074884.410833:0:39691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074884.410850:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074884.410859:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410880:0:39691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074884.410887:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074884.410900:0:39691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074884.410902:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806ae 00000080:00010000:2.0:1769074884.410906:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806ae from it ffffaa91416dfb38 00010000:00010000:2.0:1769074884.410909:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.410917:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074884.410926:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074884.410932:0:39691:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1011e8a900, open_flags 8241, mode 81b6 opened 0 00000080:00200000:2.0:1769074884.410943:0:39691:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open|creat 00000002:00010000:2.0:1769074884.410966:0:39691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:2.0:1769074884.410980:0:39691:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.411045:0:39691:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.411052:0:39691:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074884.411054:0:39691:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ae lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf306493e expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.411093:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39691 initiator: MDT0 00010000:00010000:2.0:1769074884.411100:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000020:00100000:2.0:1769074884.411112:0:39691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074884.411119:0:39691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5aa00 x1855007612442624/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074884.411129:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39691 initiator: MDT0 00000100:00100000:2.0:1769074884.411155:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442624:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074884.411312:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd900 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.415938:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442624:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074884.415978:0:39691:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416026:0:39691:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/0,1 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416036:0:39691:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/0,1 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416050:0:39691:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/0,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416062:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/0,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416075:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 5/1,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416084:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 5/1,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.416090:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074884.416103:0:39691:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=37, status=0 req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074884.416115:0:39691:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074884.416131:0:39691:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074884.416144:0:39691:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.416160:0:39691:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/280 (0 0) 00000080:00200000:2.0:1769074884.416168:0:39691:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074884.416184:0:39691:0:(namei.c:1569:ll_create_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open|creat 00000080:00200000:2.0:1769074884.416217:0:39691:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00200000:2.0:1769074884.416330:0:39691:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x118:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1769074884.416337:0:39691:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00010000:2.0:1769074884.416341:0:39691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) for lock 0x68a7113811d806b5 00000080:00010000:2.0:1769074884.416353:0:39691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00200008:2.0:1769074884.416374:0:39691:0:(file.c:1001:ll_file_open()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 101101 00000002:00100000:2.0:1769074884.416396:0:39691:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1769074884.416450:0:39691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806b5 from it ffff8b1005664060 00010000:00010000:2.0:1769074884.416454:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.416463:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074884.416490:0:39691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8), inode mode 81a4 mask 40 00000080:00200008:2.0:1769074884.416576:0:39691:0:(file.c:2648:do_file_write_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 0, count: 4 00000080:00200000:2.0:1769074884.416593:0:39691:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:2.0:1769074884.416601:0:39691:0:(file.c:2624:ll_do_tiny_write()) result: 0, original count 4 00000080:00200000:2.0:1769074884.416606:0:39691:0:(file.c:1994:ll_file_io_generic()) f11.replay-single: write ppos: 0, bytes: 4 00000080:00200000:2.0:1769074884.416612:0:39691:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:2.0:1769074884.416622:0:39691:0:(cl_io.c:214:cl_io_rw_init()) header@000000001b000fd0[0x4, 1, [0x200001b71:0x118:0x0] hash] 00000020:00200000:2.0:1769074884.416625:0:39691:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4) 0 0 00000080:00200000:2.0:1769074884.416631:0:39691:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074884.416699:0:39691:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:2.0:1769074884.416708:0:39691:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4) 4, 4 00020000:00200000:2.0:1769074884.416713:0:39691:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074884.416738:0:39691:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:2.0:1769074884.416755:0:39691:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074884.416762:0:39691:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x0 00010000:00010000:2.0:1769074884.416822:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.416833:0:39691:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.416844:0:39691:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074884.416866:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442752:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074884.416963:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.422918:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612442752:192.168.204.102@tcp:101:bash.0 00010000:00010000:2.0:1769074884.422957:0:39691:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.422979:0:39691:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.422996:0:39691:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.423026:0:39691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074884.423047:0:39691:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,2 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.423061:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,2 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.423071:0:39691:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074884.423111:0:39691:0:(vvp_io.c:1312:vvp_io_write_start()) f11.replay-single: write [0, 4) append 0 append lockpos 0 00000080:00200000:2.0:1769074884.423136:0:39691:0:(rw26.c:729:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:2.0:1769074884.423212:0:39691:0:(rw26.c:912:ll_write_end()) pos 0, len 4, copied 4 00000080:00200000:2.0:1769074884.423229:0:39691:0:(rw26.c:969:ll_write_end()) page@00000000b411f378[3 00000000e039d3b4 2 1 000000006d5f13d0] 00000080:00200000:2.0:1769074884.423463:0:39691:0:(rw26.c:969:ll_write_end()) vmpage @00000000fe54e9a6 17ffffc0001001 3:0 ffff8b10060595e8 0 lru 00000080:00200000:2.0:1769074884.423472:0:39691:0:(rw26.c:969:ll_write_end()) osc-page@00000000b3e63aab 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 0000000033af64c1 00000000bafcdbca > 3< 0 0 > 4< 0 0 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074884.423474:0:39691:0:(rw26.c:969:ll_write_end()) end page@00000000b411f378 00000080:00200000:2.0:1769074884.423476:0:39691:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:2.0:1769074884.423482:0:39691:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 1, from 0, to 4 prio 0 00000008:00010000:2.0:1769074884.423511:0:39691:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000f1d96e36 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074884.423533:0:39691:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff8b100486c478, count 1, dirtied 1 00000080:00200000:2.0:1769074884.423538:0:39691:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 1 pages 4 bytes, tot: 4 00000080:00200000:2.0:1769074884.423541:0:39691:0:(vvp_io.c:1390:vvp_io_write_start()) f11.replay-single: write bytes 0, result: 4 00000020:00010000:2.0:1769074884.423563:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.423567:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.423572:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074884.423577:0:39691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.423587:0:39691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074884.423606:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.423609:0:39691:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1005685128/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.423618:0:39691:0:(vvp_io.c:518:vvp_io_advance()) advancing 4 bytes 00000080:00200000:2.0:1769074884.423627:0:39691:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:2.0:1769074884.424103:0:39691:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.424112:0:39691:0:(file.c:2142:ll_file_io_generic()) f11.replay-single: 2 io complete with rc: 0, result: 4, restart: 0 00000080:00200000:2.0:1769074884.424124:0:39691:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000080:00200000:2.0:1769074884.424131:0:39691:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 4 00000080:00200008:2.0:1769074884.424184:0:39691:0:(file.c:424:ll_file_release()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 100001 00000002:00100000:2.0:1769074884.424877:0:39691:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074884.424903:0:39691:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074884.424910:0:39691:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5b100 x1855007612443008/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074884.424939:0:39691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612443008:192.168.204.102@tcp:35:bash.0 00000100:00100000:2.0:1769074884.426245:0:39691:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.431102:0:39691:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job bash:93598d0a-9484-4e40-94b2-393ee542cbad:39691:1855007612443008:192.168.204.102@tcp:35:bash.0 00000002:00100000:2.0:1769074884.431147:0:39691:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074884.438840:0:39885:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074884.438885:0:39885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074884.441241:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39885 initiator: MDT0 00010000:00010000:1.0:1769074884.441257:0:39885:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39885 initiator: MDT0 00010000:00010000:1.0:1769074884.441271:0:39885:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39885 initiator: MDT0 00000100:00100000:1.0:1769074884.441310:0:39885:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39885:1855007612443136:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074884.441443:0:39885:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1002c53180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.444797:0:39885:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39885:1855007612443136:192.168.204.102@tcp:101:grep.0 00010000:00010000:2.0:1769074884.444836:0:39885:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444853:0:39885:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444875:0:39885:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444901:0:39885:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444916:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444925:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444933:0:39885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074884.444946:0:39885:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006134e00 x1855007612443136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074884.444963:0:39885:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.444980:0:39885:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074884.444988:0:39885:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074884.445040:0:39885:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074884.445049:0:39885:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806c3 00000080:00010000:2.0:1769074884.445055:0:39885:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074884.445060:0:39885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806c3 from it ffffaa9141c93bf8 00010000:00010000:2.0:1769074884.445063:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.445073:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074884.445103:0:39885:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074884.445119:0:39885:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single, flags=265 00000080:00200000:2.0:1769074884.445130:0:39885:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1769074884.445141:0:39885:0:(file.c:1001:ll_file_open()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 100400 00000002:00010000:2.0:1769074884.445174:0:39885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100401 00010000:00010000:2.0:1769074884.445233:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39885 initiator: MDT0 00010000:00010000:2.0:1769074884.445420:0:39885:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39885 initiator: MDT0 00000020:00100000:2.0:1769074884.445434:0:39885:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074884.445442:0:39885:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58700 x1855007612443392/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074884.445452:0:39885:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39885 initiator: MDT0 00000100:00100000:2.0:1769074884.445476:0:39885:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39885:1855007612443392:192.168.204.102@tcp:101:grep.0 00000100:00100000:2.0:1769074884.445724:0:39885:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.450072:0:39885:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39885:1855007612443392:192.168.204.102@tcp:101:grep.0 00010000:00010000:2.0:1769074884.450116:0:39885:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x118:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.450135:0:39885:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.450149:0:39885:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074884.450163:0:39885:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074884.450179:0:39885:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074884.450193:0:39885:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 648/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074884.450209:0:39885:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074884.450224:0:39885:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.450401:0:39885:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 16 res 8589941617/280 (0 0) 00000080:00200000:2.0:1769074884.450411:0:39885:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074884.450480:0:39885:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x118:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1769074884.450486:0:39885:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) for lock 0x68a7113811d806ca 00000080:00010000:2.0:1769074884.450493:0:39885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806ca from it ffffaa9141c93bf0 00010000:00010000:2.0:1769074884.450498:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.450513:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074884.450542:0:39885:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1769074884.450673:0:39885:0:(file.c:6252:ll_getattr_dentry()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074884.450681:0:39885:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8),name=f11.replay-single 00000002:00010000:2.0:1769074884.450704:0:39885:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074884.450715:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.450727:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074884.450735:0:39885:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074884.450741:0:39885:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00010000:2.0:1769074884.450745:0:39885:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) for lock 0x68a7113811d806b5 00000080:00010000:2.0:1769074884.450749:0:39885:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806b5 from it ffffaa9141c93cd8 00010000:00010000:2.0:1769074884.450752:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074884.450758:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074884.450778:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074884.450790:0:39885:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:2.0:1769074884.450814:0:39885:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.450818:0:39885:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x400001002 00010000:00010000:2.0:1769074884.450838:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.450847:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.450860:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.450864:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.450883:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.450902:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.450915:0:39885:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.450920:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.450923:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.450927:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.450931:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.450934:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1005685128/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.450945:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074884.450965:0:39885:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 0, count: 32768 00000080:00200000:2.0:1769074884.450970:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074884.450976:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.450981:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:2.0:1769074884.450985:0:39885:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:2.0:1769074884.450992:0:39885:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.450995:0:39885:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x400001002 00010000:00010000:2.0:1769074884.451123:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451134:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.451142:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.451145:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451153:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.451161:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.451166:0:39885:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.451168:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.451171:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.451173:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1007e9b5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.451176:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.451178:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1007e9be18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.451185:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.451190:0:39885:0:(file.c:2367:file_read_confine_iter()) f11.replay-single: read [0, 32768] over eof, kms 4, file_size 4. 00000080:00200008:2.0:1769074884.451600:0:39885:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 4, count: 32768 00000080:00200000:2.0:1769074884.451609:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074884.451617:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.451621:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074884.451625:0:39885:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:2.0:1769074884.451634:0:39885:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.451636:0:39885:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x400001002 00010000:00010000:2.0:1769074884.451690:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451701:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.451710:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.451713:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451721:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.451731:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.451736:0:39885:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.451739:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.451741:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.451744:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1007e9b5a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.451746:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82238 (ffff8b1007e9be18/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.451749:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1007e9be18/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.451755:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.451759:0:39885:0:(file.c:2367:file_read_confine_iter()) f11.replay-single: read [4, 32772] over eof, kms 4, file_size 4. 00000080:00200000:2.0:1769074884.451766:0:39885:0:(file.c:1994:ll_file_io_generic()) f11.replay-single: read ppos: 4, bytes: 32768 00000080:00200000:2.0:1769074884.451771:0:39885:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:2.0:1769074884.451779:0:39885:0:(cl_io.c:214:cl_io_rw_init()) header@000000001b000fd0[0x4, 2, [0x200001b71:0x118:0x0] hash] 00000020:00200000:2.0:1769074884.451781:0:39885:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:2.0:1769074884.451784:0:39885:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074884.451799:0:39885:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:2.0:1769074884.451804:0:39885:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074884.451819:0:39885:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:2.0:1769074884.451827:0:39885:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074884.451830:0:39885:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x0 00010000:00010000:2.0:1769074884.451837:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451846:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.451855:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.452089:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.452100:0:39885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074884.452110:0:39885:0:(vvp_io.c:827:vvp_io_read_start()) f11.replay-single: read [4, 32772) 00000001:00010000:2.0:1769074884.452122:0:39885:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:2.0:1769074884.452131:0:39885:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.452133:0:39885:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef830, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074884.452140:0:39885:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.452149:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.452164:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.452167:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 8/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.452175:0:39885:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 7/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.452182:0:39885:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.452185:0:39885:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.452188:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.452190:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.452193:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef830 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.452196:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768ff8 (ffff8b1005685128/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.452198:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef830 (ffff8b1005685128/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.452204:0:39885:0:(vvp_io.c:886:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:2.0:1769074884.452214:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.452217:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.452219:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1007e9be18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074884.452224:0:39885:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.452232:0:39885:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074884.452345:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02b78 (ffff8b1005685128/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.452348:0:39885:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1005685128/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.452355:0:39885:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1769074884.452364:0:39885:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.452368:0:39885:0:(file.c:2142:ll_file_io_generic()) f11.replay-single: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1769074884.452371:0:39885:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1769074884.452401:0:39885:0:(file.c:424:ll_file_release()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 100000 00010000:00010000:2.0:1769074884.452420:0:39885:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074884.470211:0:39886:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074884.470259:0:39886:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074884.470283:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.471227:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074884.471423:0:39886:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074884.471438:0:39886:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074884.471443:0:39886:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806c3 00000080:00010000:3.0:1769074884.471537:0:39886:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806c3 from it ffffaa9141c8bbf8 00010000:00010000:3.0:1769074884.471542:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.471554:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074884.471576:0:39886:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074884.471605:0:39886:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1008809c80, open_flags 8100, mode 0 opened 0 00000080:00200000:3.0:1769074884.471614:0:39886:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:3.0:1769074884.471747:0:39886:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:3.0:1769074884.471818:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39886 initiator: MDT0 00010000:00010000:3.0:1769074884.471826:0:39886:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39886 initiator: MDT0 00000020:00100000:3.0:1769074884.471840:0:39886:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074884.471854:0:39886:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dca80 x1855007612443648/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074884.471865:0:39886:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39886 initiator: MDT0 00000100:00100000:3.0:1769074884.472075:0:39886:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39886:1855007612443648:192.168.204.102@tcp:101:grep.0 00000100:00100000:3.0:1769074884.472636:0:39886:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074884.486235:0:39886:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39886:1855007612443648:192.168.204.102@tcp:101:grep.0 00010000:00010000:3.0:1769074884.486271:0:39886:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486297:0:39886:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486305:0:39886:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486331:0:39886:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486342:0:39886:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486356:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486363:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074884.486370:0:39886:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074884.486382:0:39886:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b10037dca80 x1855007612443648/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074884.486397:0:39886:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037dca80 x1855007612443648/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074884.486410:0:39886:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 0 00000002:00010000:3.0:1769074884.486431:0:39886:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074884.486442:0:39886:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:3.0:1769074884.487855:0:39886:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074884.487957:0:39886:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffff8b1004859148): rc = 0 00000080:00200000:3.0:1769074884.487977:0:39886:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 0 -> 0 00000080:00010000:3.0:1769074884.487985:0:39886:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d806d1 00000080:00010000:3.0:1769074884.487999:0:39886:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00200008:3.0:1769074884.488019:0:39886:0:(file.c:1001:ll_file_open()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 100400 00000002:00100000:3.0:1769074884.488055:0:39886:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037dca80 x1855007612443648/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1769074884.488126:0:39886:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806d1 from it ffff8b10065d7c00 00010000:00010000:3.0:1769074884.488133:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074884.488146:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074884.488177:0:39886:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148), inode mode 81a4 mask 44 00000080:00200008:3.0:1769074884.488208:0:39886:0:(file.c:6252:ll_getattr_dentry()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074884.488215:0:39886:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148),name=f11.replay-single-2 00000002:00010000:3.0:1769074884.488238:0:39886:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x115:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074884.488251:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074884.488267:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074884.488274:0:39886:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074884.488279:0:39886:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00010000:3.0:1769074884.488282:0:39886:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d806d1 00000080:00010000:3.0:1769074884.488285:0:39886:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806d1 from it ffffaa9141c8bcd8 00010000:00010000:3.0:1769074884.488288:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074884.488294:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074884.488325:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074884.488337:0:39886:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:3.0:1769074884.488395:0:39886:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074884.488400:0:39886:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76930, osc lock ffff8b10036171d0, flags 0x400001002 00010000:00010000:3.0:1769074884.488422:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074884.488432:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074884.488454:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074884.488458:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074884.488471:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074884.488483:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074884.488495:0:39886:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 1 00000020:00010000:3.0:1769074884.488500:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:3.0:1769074884.488504:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074884.488508:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76930 (ffff8b10056851b8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074884.490156:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074884.490169:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76930 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074884.490193:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1769074884.490228:0:39886:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 0, count: 32768 00000080:00200000:3.0:1769074884.490237:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074884.490253:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074884.490259:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074884.490264:0:39886:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1769074884.490737:0:39886:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.490744:0:39886:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76930, osc lock ffff8b1010ac6260, flags 0x400001002 00010000:00010000:2.0:1769074884.490773:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.490793:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.490809:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.490813:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.490823:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.490840:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.490849:0:39886:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.490854:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.490857:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.490859:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76930 (ffff8b1005685008/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.490863:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.490956:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76930 (ffff8b10056851b8/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.490968:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.490975:0:39886:0:(file.c:2367:file_read_confine_iter()) f11.replay-single-2: read [0, 32768] over eof, kms 4, file_size 4. 00000080:00200008:2.0:1769074884.491113:0:39886:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 4, count: 32768 00000080:00200000:2.0:1769074884.491119:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074884.491127:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.491131:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1769074884.491134:0:39886:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1769074884.491143:0:39886:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.491145:0:39886:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x400001002 00010000:00010000:2.0:1769074884.491154:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.491165:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.491173:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.491176:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.491184:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.491193:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.491198:0:39886:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.491201:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.491204:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.491207:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b1005685008/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.491211:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768db0 (ffff8b10056851b8/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.491213:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b10056851b8/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.491219:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.491223:0:39886:0:(file.c:2367:file_read_confine_iter()) f11.replay-single-2: read [4, 32772] over eof, kms 4, file_size 4. 00000080:00200000:2.0:1769074884.491231:0:39886:0:(file.c:1994:ll_file_io_generic()) f11.replay-single-2: read ppos: 4, bytes: 32768 00000080:00200000:2.0:1769074884.491236:0:39886:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single-2: desiginated mirror: 0 00000020:00200000:2.0:1769074884.491245:0:39886:0:(cl_io.c:214:cl_io_rw_init()) header@000000007469e5a4[0x4, 2, [0x200001b71:0x115:0x0] hash] 00000020:00200000:2.0:1769074884.491247:0:39886:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:2.0:1769074884.491250:0:39886:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074884.491265:0:39886:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:2.0:1769074884.491270:0:39886:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074884.492704:0:39886:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:2.0:1769074884.492732:0:39886:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074884.492738:0:39886:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef230, osc lock ffff8b1010ac6260, flags 0x0 00010000:00010000:2.0:1769074884.492757:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.492773:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.492787:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.492797:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.492805:0:39886:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074884.492820:0:39886:0:(vvp_io.c:827:vvp_io_read_start()) f11.replay-single-2: read [4, 32772) 00000001:00010000:2.0:1769074884.492840:0:39886:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1769074884.492845:0:39886:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074884.492847:0:39886:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b10034ef830, osc lock ffff8b1010ac7a20, flags 0x400001002 00010000:00010000:2.0:1769074884.492855:0:39886:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.492864:0:39886:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.493226:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074884.493230:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 8/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.493237:0:39886:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 7/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074884.493243:0:39886:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074884.493252:0:39886:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 1 00000020:00010000:2.0:1769074884.493256:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.493259:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.493262:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef830 (ffff8b10056851b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.493265:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768000 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.493268:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef830 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.493281:0:39886:0:(vvp_io.c:886:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:2.0:1769074884.493295:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e023f8 (ffff8b1005685248/0) at cl_lock_release():245 00000020:00010000:2.0:1769074884.493298:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e023f8 (ffff8b1005685248/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074884.493302:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b10034ef230 (ffff8b10056851b8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074884.493592:0:39886:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074884.493604:0:39886:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074884.493622:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e023f8 (ffff8b1005685248/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074884.493625:0:39886:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b10034ef230 (ffff8b1005685248/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074884.493633:0:39886:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1769074884.493975:0:39886:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074884.493981:0:39886:0:(file.c:2142:ll_file_io_generic()) f11.replay-single-2: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1769074884.493985:0:39886:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1769074884.494016:0:39886:0:(file.c:424:ll_file_release()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 100000 00000002:00100000:2.0:1769074884.494102:0:39886:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10037dca80 x1855007612443648/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074884.494116:0:39886:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074884.494122:0:39886:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5b480 x1855007612443904/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074884.494148:0:39886:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39886:1855007612443904:192.168.204.102@tcp:35:grep.0 00000100:00100000:2.0:1769074884.494343:0:39886:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cdb40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074884.504581:0:39886:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:39886:1855007612443904:192.168.204.102@tcp:35:grep.0 00000002:00100000:2.0:1769074884.504627:0:39886:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10037dca80 x1855007612443648/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1769074885.151996:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074886.176479:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074887.200290:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074888.227169:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074889.248936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074889.380132:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444160:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074889.381160:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444416:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074889.381222:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074889.381242:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612444032:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074889.381281:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444288:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074889.392763:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444288:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074889.392788:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444416:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074890.271912:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074891.297726:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074892.320618:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074893.343920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074894.367922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074894.496728:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612444544:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074894.496875:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444800:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074894.497087:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074894.497096:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444672:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074894.497124:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074894.505764:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444928:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074894.505790:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444800:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074895.392671:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074896.418047:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074897.440112:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074898.464140:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074899.488507:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074899.616705:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612445056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074899.618625:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769074899.618680:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612445184:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074899.620422:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445440:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074899.620809:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612445312:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074899.625499:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612445312:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074899.631815:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445440:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074900.512017:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074901.546637:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074902.560722:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074903.585146:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074904.622371:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074904.736034:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445696:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074904.736153:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445952:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074904.736332:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074904.736342:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612445568:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074904.736383:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445824:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074904.748535:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445824:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074904.748573:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445952:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074905.632136:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074905.761407:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074889/real 1769074889] req@ffff8b1006026680 x1855007612444032/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074905 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074905.761446:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006026680 x1855007612444032/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074905 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074905.761457:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1769074905.761462:0:2365:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1769074905.761466:0:2365:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1769074905.761469:0:2365:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1769074905.761474:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006026680 x1855007612444032/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074905 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074905.761483:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006025880 x1855007612444544/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074910 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074905.761505:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5b480 x1855007612445056/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074915 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074905.761538:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037dca80 x1855007612445568/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074920 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1769074905.761557:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1769074905.761560:0:2365:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1769074905.761562:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1769074905.761565:0:2365:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:1.0:1769074905.761581:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074889/real 1769074889] req@ffff8b1006025180 x1855007612444160/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074905 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074905.761595:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006025180 x1855007612444160/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074905 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074905.761602:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074905.761606:0:2365:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074905.761611:0:2365:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074905.761627:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612444032:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074905.761636:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612444160:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074905.761732:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612445056:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074905.761763:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612445568:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074905.762038:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612444544:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074905.762078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074905.762084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074905.762088:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074905.762102:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1840 00000100:00080000:0.0:1769074905.762119:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769074905.762154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612446080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074905.762183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074905.762417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074905.762420:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074905.762426:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1856 00000100:00080000:0.0:1769074905.762434:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074905.762445:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074905.762459:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612446208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074905.762515:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074905.762633:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074905.763128:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074905.763461:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612446080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769074905.763519:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612446208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074905.769948:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074905.769972:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446336:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074906.656675:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074907.682920:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074908.703987:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074909.727882:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074910.367953:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074910.367977:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:2.0:1769074910.368012:0:37:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000607cb37b ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074910.368037:0:37:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000607cb37b ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074910.368115:0:37:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00100000:2.0:1769074910.368219:0:37:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1005d59c00 - object 0x280000400:770 offset 0<>4 00000080:00200000:2.0:1769074910.368276:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1769074910.368390:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612446592:192.168.204.102@tcp:4:grep.0 00000008:00010000:3.0:1769074910.373607:0:2367:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000607cb37b ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:3.0:1769074910.373710:0:2367:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000607cb37b ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000100:00100000:3.0:1769074910.373748:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612446592:192.168.204.102@tcp:4:grep.0 00010000:00010000:3.0:1769074910.753799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074910.880610:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074894/real 1769074894] req@ffff8b1006026300 x1855007612444672/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074910 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074910.880676:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006026300 x1855007612444672/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074910 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074910.880686:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 65, was 64): CONNECTING 00000100:00100000:0.0:1769074910.880703:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612444672:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074910.880778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074910.880786:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769074910.880788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074910.880792:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769074910.880953:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612446720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074910.881076:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1769074910.881083:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612446848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074910.893884:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612446848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074910.893909:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612446720:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074911.776241:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074912.802739:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074913.824197:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074914.847948:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769074915.496544:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074899/real 1769074899] req@ffff8b1005d58e00 x1855007612445184/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074915.496584:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58e00 x1855007612445184/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074915.496594:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 65, was 64): CONNECTING 00000100:00100000:3.0:1769074915.496615:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612445184:192.168.204.102@tcp:400:kworker.0 00000080:00200000:1.0:1769074915.496964:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074915.496980:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1769074915.497013:0:37:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000f1d96e36 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074915.497032:0:37:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000f1d96e36 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074915.497092:0:37:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00100000:1.0:1769074915.498793:0:37:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b1006134700 - object 0x240000400:770 offset 0<>4 00000080:00200000:1.0:1769074915.498882:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1769074915.499510:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446976:192.168.204.102@tcp:4:grep.0 00000008:00010000:1.0:1769074915.537845:0:2364:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000f1d96e36 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074915.537905:0:2364:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000f1d96e36 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074915.537944:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612446976:192.168.204.102@tcp:4:grep.0 00010000:00010000:0.0:1769074915.872152:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074916.000074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074916.000090:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769074916.000093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074916.000097:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769074916.000217:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612447232:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074916.001722:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074905/real 1769074905] req@ffff8b1006025880 x1855007612446080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074916.001757:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612446080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074916.001767:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074905/real 1769074905] req@ffff8b1006024000 x1855007612446208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074916.001778:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612446208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074915 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074916.001811:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074916.001841:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074916.001846:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612446080:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769074916.001852:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074916.001858:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074916.001861:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612446208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074916.002311:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074916.002334:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612447104:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074916.002628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074916.002634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074916.002657:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074916.002664:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1898 00000100:00080000:0.0:1769074916.002672:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1769074916.002680:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769074916.002711:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612447360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074916.002738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074916.002742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074916.002744:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074916.002748:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1898 00000100:00080000:0.0:1769074916.002751:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1769074916.002755:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074916.002766:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074916.002776:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612447488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074916.008604:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612447360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769074916.008798:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447488:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769074916.022208:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:2.0:1769074916.022241:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1769074916.022254:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3064428 to 0x143201ebf3064b3d 00000100:00080000:2.0:1769074916.022261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1769074916.022270:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:3.0:1769074916.023374:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612447360:192.168.204.102@tcp:250:kworker.0 00010000:00080000:3.0:1769074916.023427:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1006024a80 x1855007612447488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074931 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074916.023442:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280521 gen 1 00000100:00100000:3.0:1769074916.023494:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1769074916.023507:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 23 00000100:00080000:3.0:1769074916.023526:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1769074916.023533:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074916.023535:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280521 gen 1 00000100:00100000:3.0:1769074916.023540:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1769074916.023547:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 94489280521 last 0 00000100:00080000:3.0:1769074916.023555:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074900 ref 1 fl New:RQU/604/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1769074916.023567:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769074916.023596:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612442624:192.168.204.102@tcp:101:bash.0 00000100:00080000:2.0:1769074916.023911:0:40084:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:2.0:1769074916.023916:0:40084:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:2.0:1769074916.023921:0:40084:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:2.0:1769074916.023952:0:40084:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074916.023965:0:40084:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074916.023972:0:40084:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074916.023993:0:40084:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074916.024003:0:40084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d80630 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064436 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074916.024018:0:40084:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074916.024025:0:40084:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074916.024031:0:40084:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074916.024035:0:40084:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074916.024038:0:40084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005699a00/0x68a7113811d8063e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064444 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074916.024054:0:40084:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074916.024060:0:40084:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074916.024066:0:40084:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1769074916.024069:0:40084:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074916.024073:0:40084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169600/0x68a7113811d80645 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf30645b7 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074916.024089:0:40084:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:2.0:1769074916.024097:0:40084:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074916.024102:0:40084:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1769074916.024105:0:40084:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074916.024109:0:40084:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698c00/0x68a7113811d80637 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf306443d expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:2.0:1769074916.024129:0:40084:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:2.0:1769074916.024137:0:40084:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:2.0:1769074916.024140:0:40084:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1769074916.024142:0:40084:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:2.0:1769074916.024146:0:40084:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00010000:1.0:1769074916.025142:0:2364:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1769074916.025159:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612447104:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074916.025203:0:2364:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1769074916.025207:0:2364:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff8b101127b800: committing for last_committed 4294967569 gen 10 00000100:00100000:1.0:1769074916.025226:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612447232:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074916.025258:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074916.025276:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074916.025282:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074916.025284:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074916.025286:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074916.025976:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769074916.128004:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/608 e 0 to 0 dl 1769074932 ref 2 fl Interpret:RQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000002:00080000:3.0:1769074916.128035:0:2363:0:(mdc_request.c:714:mdc_replay_open()) @@@ updating close body with new fh req@ffff8b1005d5b100 x1855007612443008/t94489280523(94489280523) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074900 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000100:00020000:3.0:1769074916.128044:0:2363:0:(client.c:3418:ptlrpc_replay_interpret()) @@@ status 301, old was 0 req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/608 e 0 to 0 dl 1769074932 ref 2 fl Interpret:RQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1769074916.165245:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1769074916.165257:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280521 gen 1 00000100:00100000:3.0:1769074916.165278:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/608 e 0 to 0 dl 1769074932 ref 2 fl Interpret:RQU/604/0 rc 301/301 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1769074916.165288:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 94489280521 last 94489280522 00000100:00080000:3.0:1769074916.165300:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d5b100 x1855007612443008/t94489280523(94489280523) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074900 ref 2 fl New:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1769074916.165316:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612442624:192.168.204.102@tcp:101:bash.0 00000100:00100000:2.0:1769074916.166085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443008:192.168.204.102@tcp:35:bash.0 00000100:00080000:2.0:1769074916.188832:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d5b100 x1855007612443008/t94489280523(94489280523) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074932 ref 3 fl Interpret:RQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769074916.188852:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1769074916.188856:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280521 gen 1 00000100:00100000:2.0:1769074916.188863:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/608 e 0 to 0 dl 1769074932 ref 1 fl Complete:RQU/604/0 rc 0/301 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769074916.188873:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 94489280521 last 94489280523 00000100:00080000:2.0:1769074916.188883:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074900 ref 1 fl New:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074916.188903:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443008:192.168.204.102@tcp:35:bash.0 00000100:00100000:2.0:1769074916.188932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443392:192.168.204.102@tcp:101:grep.0 00000100:00080000:2.0:1769074916.207736:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074932 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000002:00080000:2.0:1769074916.207760:0:2363:0:(mdc_request.c:695:mdc_replay_open()) updating handle from 0x143201ebf306498b to 0x143201ebf3064b67 00000100:00020000:2.0:1769074916.207766:0:2363:0:(client.c:3418:ptlrpc_replay_interpret()) @@@ status 301, old was 0 req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074932 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769074916.207775:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1769074916.207778:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280521 gen 1 00000100:00100000:2.0:1769074916.207785:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d5aa00 x1855007612442624/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/608 e 0 to 0 dl 1769074932 ref 1 fl Complete:RQU/604/0 rc 0/301 job:'bash.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769074916.207792:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 94489280521 last 94489280524 00000100:00080000:2.0:1769074916.207798:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:2.0:1769074916.208974:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443392:192.168.204.102@tcp:101:grep.0 00010000:00080000:0.0:1769074916.209136:0:40085:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:0.0:1769074916.209143:0:40085:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (5) 00000080:00200000:0.0:1769074916.209202:0:40085:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x115:0x0]: losing layout lock 00010000:00010000:0.0:1769074916.209260:0:40085:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006168c00/0x68a7113811d806d1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf30649a7 expref: -99 pid: 39886 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074916.209292:0:40085:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b9200/0x68a7113811d806c3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0x143201ebf3064976 expref: -99 pid: 39885 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074916.209318:0:40085:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074916.209339:0:40085:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698a00/0x68a7113811d806b5 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf306495a expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074916.209368:0:40085:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x115:0x0]: losing layout lock 00010000:00010000:0.0:1769074916.209376:0:40085:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699200/0x68a7113811d80676 lrc: 0/0,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf306486c expref: -99 pid: 39691 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074916.209388:0:40085:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00010000:00010000:0.0:1769074916.210495:0:40085:0:(ldlm_request.c:2606:replay_one_lock()) ### replaying lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064984 expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000100:00100000:3.0:1769074916.210919:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447616:192.168.204.102@tcp:101:ldlm_lock_repla.0 00010000:00080000:0.0:1769074916.211333:0:40085:0:(ldlm_request.c:2656:lock_can_replay()) check lock replay limit, inflights = 1(8) 00010000:00010000:2.0:1769074916.225180:0:2363:0:(ldlm_request.c:2523:replay_lock_interpret()) ### replayed lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064b6e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000100:00080000:2.0:1769074916.225208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:2.0:1769074916.225266:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447616:192.168.204.102@tcp:101:ldlm_lock_repla.0 00000100:00100000:2.0:1769074916.225308:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447744:192.168.204.102@tcp:400:ptlrpcd_rcv.0 00000100:00100000:2.0:1769074916.355695:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 94489280524 gen 1 00000100:00100000:2.0:1769074916.355748:0:2363:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074932 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769074916.355766:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:2.0:1769074916.355773:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1005d59880 x1855007612447744/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074932 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074916.355782:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1005d59880 x1855007612447744/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074932 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074916.355790:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:3.0:1769074916.356745:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00100000:3.0:1769074916.356756:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612447744:192.168.204.102@tcp:400:ptlrpcd_rcv.0 00010000:00010000:0.0:1769074916.907681:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1769074917.471868:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1769074917.471881:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 28 10000000:01000000:3.0:1769074917.471889:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1769074917.471970:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.471985:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.471998:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074917.472036:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612447872:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.472172:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.478615:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612447872:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074917.479859:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.479882:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074917.480030:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448000:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.480171:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.491423:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448000:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:2.0:1769074917.491593:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:2.0:1769074917.491846:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1769074917.491860:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074917.491872:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074917.491883:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1769074917.491887:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:2.0:1769074917.491890:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1769074917.491945:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074917.491954:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074917.491978:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074917.492003:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448128:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074917.492114:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074917.495098:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448128:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074917.495215:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074917.495234:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074917.495354:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448256:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074917.495501:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.510085:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448256:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.510191:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448384:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.510324:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.523482:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448384:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1769074917.529554:0:40125:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:0.0:1769074917.529994:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:0.0:1769074917.530022:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1769074917.530036:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.530050:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074917.530061:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:0.0:1769074917.530063:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:0.0:1769074917.530066:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:0.0:1769074917.530117:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.530125:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.530148:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074917.530192:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448512:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074917.530348:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.545601:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448512:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074917.545680:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.545696:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074917.545804:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448640:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.545898:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 11 seconds 00000100:00100000:1.0:1769074917.563416:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448640:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1769074917.563496:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448768:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1769074917.563599:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10097a96c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074917.576032:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448768:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1769074917.593392:0:40126:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:0.0:1769074917.593750:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:0.0:1769074917.593777:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:0.0:1769074917.593792:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.593805:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074917.593816:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1769074917.593819:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:0.0:1769074917.593822:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1769074917.593873:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.593882:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.593904:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074917.593938:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448896:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074917.594413:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10037ec480 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.599041:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612448896:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074917.599083:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.599099:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074917.599216:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612449024:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074917.599314:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364cb40 going to sleep for 11 seconds 00000100:00100000:3.0:1769074917.605106:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612449024:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1769074917.605182:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1769074917.605196:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074917.605209:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074917.921593:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074918.943925:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074919.968921:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769074920.036038:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074904/real 1769074904] req@ffff8b10037de300 x1855007612445696/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074920 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074920.036087:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037de300 x1855007612445696/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074920 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074920.036116:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612445696:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074920.992697:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074921.121507:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449280:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.121676:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449536:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.121720:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074921.121728:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612449152:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.121763:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449408:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.136352:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612449152:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.136377:0:2364:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8b101127b800: committing for last_committed 8589934865 gen 10 00000100:00100000:1.0:1769074921.136399:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449408:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.136412:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449280:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074921.136424:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612449536:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074922.016737:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074923.039848:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074924.064158:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1769074924.753557:0:40178:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074924.753570:0:40178:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074924.753592:0:40178:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074924.753682:0:40178:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 40178 initiator: MDT0 00010000:00010000:2.0:1769074924.753764:0:40178:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 40178 initiator: MDT0 00010000:00010000:2.0:1769074924.753776:0:40178:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 40178 initiator: MDT0 00000100:00100000:2.0:1769074924.753802:0:40178:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449664:192.168.204.102@tcp:101:lfs.0 00000100:00100000:2.0:1769074924.753894:0:40178:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074924.761903:0:40178:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449664:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074924.761930:0:40178:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.761960:0:40178:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.761972:0:40178:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.761984:0:40178:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.761996:0:40178:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.762004:0:40178:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.762012:0:40178:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074924.762022:0:40178:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d5aa00 x1855007612449664/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074940 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074924.762035:0:40178:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.762048:0:40178:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074924.762056:0:40178:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074924.762082:0:40178:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074924.762089:0:40178:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:2.0:1769074924.762094:0:40178:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074924.762098:0:40178:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91420bfcb8 00010000:00010000:2.0:1769074924.762102:0:40178:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.762110:0:40178:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074924.763271:0:40178:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074924.763280:0:40178:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074924.763297:0:40178:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074924.763308:0:40178:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.763321:0:40178:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074924.763329:0:40178:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074924.763334:0:40178:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074924.763337:0:40178:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:2.0:1769074924.763342:0:40178:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91420bfcb8 00010000:00010000:2.0:1769074924.763346:0:40178:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.763354:0:40178:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074924.763457:0:40178:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074924.763465:0:40178:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074924.763472:0:40178:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.763481:0:40178:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074924.763487:0:40178:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074924.763491:0:40178:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074924.763494:0:40178:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:2.0:1769074924.763498:0:40178:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91420bfbb0 00010000:00010000:2.0:1769074924.763502:0:40178:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074924.763509:0:40178:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074924.763517:0:40178:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074924.763524:0:40178:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074924.763599:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.763613:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00800000:00000004:2.0:1769074924.763619:0:40178:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1874, max_age 1916 00000100:00100000:2.0:1769074924.763808:0:40178:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449792:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074924.763891:0:40178:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074924.768860:0:40178:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449792:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074924.768888:0:40178:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00200080:2.0:1769074924.769024:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769041:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769048:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769052:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769056:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769059:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769063:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769067:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769071:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769075:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769079:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769082:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769086:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769089:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769093:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769097:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769101:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769104:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769108:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769111:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769115:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769119:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769122:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769126:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769129:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769132:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769136:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769140:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769144:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769147:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769151:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769154:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769158:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769161:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769165:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769168:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769186:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769191:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769195:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769198:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769202:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769206:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769210:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769226:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769231:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769235:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769239:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769242:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769246:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769250:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769254:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769257:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769261:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769265:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769269:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769273:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769276:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769280:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769284:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769288:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769292:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769295:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769299:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769302:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769306:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769310:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769313:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769317:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769321:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769324:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769328:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769332:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769335:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769339:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769343:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769346:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769350:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769354:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769358:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769362:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769365:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769369:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769373:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769376:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769380:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769383:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769387:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769390:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769394:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769397:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769402:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769406:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769410:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769413:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769416:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769420:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769423:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769426:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769430:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769433:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769436:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769440:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769444:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769448:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769451:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769455:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769459:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769462:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769466:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769469:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769473:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769476:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769480:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769483:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769487:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769490:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769493:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769496:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769500:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769503:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769507:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769510:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769514:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769518:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769522:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769525:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769529:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00800000:00000080:2.0:1769074924.769532:0:40178:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00000080:00200080:2.0:1769074924.769537:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:2.0:1769074924.769542:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00020000:00000004:2.0:1769074924.769549:0:40178:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1874, max_age 1916 00000100:00100000:2.0:1769074924.769586:0:40178:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449920:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074924.769694:0:40178:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074924.773451:0:40178:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612449920:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074924.773482:0:40178:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:2.0:1769074924.775374:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:2.0:1769074924.775397:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a8800 uarg=0000000000000000 00020000:00000004:2.0:1769074924.775501:0:40178:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1874, max_age 1916 00000100:00100000:2.0:1769074924.775547:0:40178:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612450048:192.168.204.102@tcp:13:lfs.0 00000100:00100000:3.0:1769074924.776341:0:40178:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074924.784788:0:40178:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40178:1855007612450048:192.168.204.102@tcp:13:lfs.0 00020000:00000004:3.0:1769074924.784992:0:40178:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:3.0:1769074924.785378:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785396:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785405:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785409:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785412:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785415:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785419:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785422:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785425:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785428:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785431:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785434:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785437:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785440:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785443:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785446:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785449:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785452:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785455:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785458:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785461:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785464:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785468:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785471:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785474:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785477:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785481:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785485:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785488:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785491:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785495:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785498:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785501:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785503:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785506:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785509:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785512:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785516:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785520:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785523:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785527:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785530:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785534:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785536:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785539:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785543:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785546:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785549:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785553:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785555:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785559:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785563:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785567:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785571:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785575:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785578:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785581:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785585:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785589:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785592:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785595:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785598:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785602:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785604:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785608:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785611:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785615:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785619:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785624:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785627:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785631:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785635:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785701:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785707:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785711:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785715:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785719:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785723:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785727:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785730:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785734:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785737:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785740:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785744:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785748:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785751:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785754:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785757:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785761:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785764:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785769:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785772:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785776:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785779:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785784:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785787:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785791:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785793:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785797:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.785799:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.785802:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786444:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786460:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786467:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786471:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786474:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786477:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786480:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786483:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786486:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786489:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786492:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786495:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786497:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786501:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786503:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786507:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786510:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786513:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786516:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786519:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786522:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786525:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786528:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200080:3.0:1769074924.786531:0:40178:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe05460bd0 00020000:00000080:3.0:1769074924.786534:0:40178:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007f35400 uarg=0000000000000000 00000080:00200008:3.0:1769074924.786547:0:40178:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:0.0:1769074925.089955:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074926.111972:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074926.240623:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612450176:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.240764:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450432:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.240800:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074926.241941:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.242234:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.250049:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450304:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.250066:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450560:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.250077:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612450432:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074926.251669:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612450176:192.168.204.102@tcp:400:kworker.0 00010000:00010000:1.0:1769074927.135740:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074928.165457:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074928.193243:0:40428:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00010000:00010000:0.0:1769074929.183836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074929.568193:0:40582:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:0.0:1769074930.208697:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074931.235995:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074931.368610:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612450688:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.368730:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612450944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.368764:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074931.368771:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612450816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.368796:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612451072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.377999:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612451072:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.378783:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612450816:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.378798:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612450944:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.393480:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612450688:192.168.204.102@tcp:400:kworker.0 00000080:00200008:1.0:1769074931.602172:0:40620:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074931.602189:0:40620:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074931.602215:0:40620:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074931.602236:0:40620:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.602264:0:40620:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074931.602273:0:40620:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074931.602282:0:40620:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074931.602286:0:40620:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074931.602314:0:40620:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91414abcb8 00010000:00010000:1.0:1769074931.602319:0:40620:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.602328:0:40620:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074931.603355:0:40620:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074931.603362:0:40620:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074931.603377:0:40620:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074931.603388:0:40620:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.603398:0:40620:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074931.603404:0:40620:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074931.603409:0:40620:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074931.603412:0:40620:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074931.603487:0:40620:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91414abcb8 00010000:00010000:1.0:1769074931.603493:0:40620:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.603502:0:40620:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074931.603615:0:40620:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074931.603624:0:40620:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074931.603631:0:40620:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.603887:0:40620:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074931.603896:0:40620:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074931.603901:0:40620:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074931.603904:0:40620:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074931.603908:0:40620:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa91414abbb0 00010000:00010000:1.0:1769074931.603911:0:40620:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074931.603917:0:40620:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074931.603926:0:40620:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:1.0:1769074931.603933:0:40620:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:1.0:1769074931.604036:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:1.0:1769074931.604052:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b1007c41400 uarg=0000000000000000 00800000:00000004:1.0:1769074931.604059:0:40620:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1917, max_age 1923 00000100:00100000:1.0:1769074931.604201:0:40620:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451200:192.168.204.102@tcp:41:lfs.0 00000100:00100000:0.0:1769074931.604802:0:40620:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56f00 going to sleep for 11 seconds 00000100:00100000:0.0:1769074931.609839:0:40620:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451200:192.168.204.102@tcp:41:lfs.0 00800000:00000004:0.0:1769074931.609871:0:40620:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00200080:0.0:1769074931.609962:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.609980:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.609988:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.609992:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.609996:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610000:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610004:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610007:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610011:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610015:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610019:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610022:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610026:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610030:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610034:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610037:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610041:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610044:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610048:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610051:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610054:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610057:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610060:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610063:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610066:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610068:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610071:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610074:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610077:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610080:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610083:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610086:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610090:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610093:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610109:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610114:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610119:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610122:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610125:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610128:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610131:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610134:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610138:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610141:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610144:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610148:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610152:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610155:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610159:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610162:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610165:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610168:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610172:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610175:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610178:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610182:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610185:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610189:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610193:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610197:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610200:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610203:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610207:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610211:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610214:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610218:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610222:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610226:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610230:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610233:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610237:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610240:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610243:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610247:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610251:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610254:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610257:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610260:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610264:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610267:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610272:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610275:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610279:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610282:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610286:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610289:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610292:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610295:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610299:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610303:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610307:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610310:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610314:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610318:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610322:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610325:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610329:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610332:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610336:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610339:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610343:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610346:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610349:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610353:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610356:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610359:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610363:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610366:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610369:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610372:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610376:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610379:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610382:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610385:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610388:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610391:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610395:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610398:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610402:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610405:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610408:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610411:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610415:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610418:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610422:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610425:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610429:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00800000:00000080:0.0:1769074931.610433:0:40620:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.610436:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.610442:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00020000:00000004:0.0:1769074931.610449:0:40620:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1917, max_age 1923 00000100:00100000:0.0:1769074931.610487:0:40620:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451328:192.168.204.102@tcp:13:lfs.0 00000100:00100000:0.0:1769074931.610561:0:40620:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10097296c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074931.614886:0:40620:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451328:192.168.204.102@tcp:13:lfs.0 00020000:00000004:0.0:1769074931.615260:0:40620:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:0.0:1769074931.615343:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.615359:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00020000:00000004:0.0:1769074931.615367:0:40620:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1917, max_age 1923 00000100:00100000:0.0:1769074931.615404:0:40620:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451456:192.168.204.102@tcp:13:lfs.0 00000100:00100000:0.0:1769074931.615478:0:40620:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10097296c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074931.620062:0:40620:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:40620:1855007612451456:192.168.204.102@tcp:13:lfs.0 00020000:00000004:0.0:1769074931.620091:0:40620:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:0.0:1769074931.620176:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620194:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620201:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620206:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620210:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620214:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620217:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620220:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620223:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620227:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620230:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620233:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620236:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620238:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620242:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620246:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620249:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620252:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620255:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620258:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620261:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620265:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620268:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620272:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620532:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620540:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620545:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620548:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620552:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620555:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620559:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620562:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620566:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620568:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620572:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620575:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620577:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620580:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620583:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620586:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620589:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620593:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620596:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620599:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620602:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620605:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620609:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620612:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620616:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620619:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620623:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620626:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620631:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620634:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620687:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620693:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620698:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620702:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620705:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620709:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620713:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:0.0:1769074931.620716:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b100602bc00 uarg=0000000000000000 00000080:00200080:0.0:1769074931.620719:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.620999:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621011:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621016:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621020:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621024:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621028:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621031:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621035:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621039:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621043:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621046:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621048:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621050:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621053:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621055:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621057:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621059:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621061:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621063:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621065:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621067:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621068:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621070:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621072:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621074:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621075:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621077:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621079:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621081:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621082:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621084:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621086:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621088:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621089:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621091:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621093:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621094:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621107:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621112:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621117:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621121:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621125:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621128:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621132:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621136:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621140:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621143:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621147:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621150:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621153:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621156:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621159:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621163:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621166:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621170:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621173:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621177:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621180:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621183:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621187:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621190:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200080:2.0:1769074931.621194:0:40620:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fffc6d30ef0 00020000:00000080:2.0:1769074931.621197:0:40620:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096a9800 uarg=0000000000000000 00000080:00200008:2.0:1769074931.621217:0:40620:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:0.0:1769074931.655054:0:40636:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074931.655084:0:40636:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074931.655114:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074931.655130:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074931.655138:0:40636:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074931.655145:0:40636:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:0.0:1769074931.655150:0:40636:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:0.0:1769074931.655157:0:40636:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa9141067bf8 00010000:00010000:0.0:1769074931.655161:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074931.655169:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074931.655184:0:40636:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:0.0:1769074931.655198:0:40636:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1009033380, open_flags 8100, mode 0 opened 0 00000080:00200000:0.0:1769074931.655204:0:40636:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:0.0:1769074931.655219:0:40636:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:0.0:1769074931.655378:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 40636 initiator: MDT0 00010000:00010000:0.0:1769074931.655387:0:40636:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40636 initiator: MDT0 00000020:00100000:0.0:1769074931.655400:0:40636:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074931.655413:0:40636:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006024380 x1855007612451712/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074931.655422:0:40636:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40636 initiator: MDT0 00000100:00100000:0.0:1769074931.655447:0:40636:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40636:1855007612451712:192.168.204.102@tcp:101:grep.0 00000100:00100000:0.0:1769074931.660168:0:40636:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10097296c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074931.671746:0:40636:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40636:1855007612451712:192.168.204.102@tcp:101:grep.0 00010000:00010000:3.0:1769074931.671771:0:40636:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671788:0:40636:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671796:0:40636:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671807:0:40636:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671821:0:40636:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671831:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671838:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.671844:0:40636:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074931.671854:0:40636:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1006024380 x1855007612451712/t98784247809(98784247809) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074931.671865:0:40636:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006024380 x1855007612451712/t98784247809(98784247809) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074931.671877:0:40636:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 0 00000002:00010000:3.0:1769074931.671889:0:40636:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.671898:0:40636:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/280 (0 0) 00000080:00200000:3.0:1769074931.671904:0:40636:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074931.671982:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1769074931.671994:0:40636:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074931.672027:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:3.0:1769074931.672085:0:40636:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 1 [cma]time: 0 0 0 by oinfo size 4 blocks 1 [cma]time 1769074884 1769074884 1769074884 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 1/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074931.672151:0:40636:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x118:0x0](ffff8b100486c2c8): rc = 0 00000080:00200000:3.0:1769074931.672166:0:40636:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x118:0x0]: layout version change: 0 -> 65535 00000080:00010000:3.0:1769074931.672172:0:40636:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) for lock 0x68a7113811d806fb 00000080:00010000:3.0:1769074931.672183:0:40636:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00200008:3.0:1769074931.672197:0:40636:0:(file.c:1001:ll_file_open()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 100400 00000002:00100000:3.0:1769074931.672217:0:40636:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1006024380 x1855007612451712/t98784247809(98784247809) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074931.672252:0:40636:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1006024380 x1855007612451712/t98784247809(98784247809) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1769074931.672262:0:40636:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1769074931.672268:0:40636:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de300 x1855007612451968/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1769074931.672287:0:40636:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40636:1855007612451968:192.168.204.102@tcp:35:grep.0 00000100:00100000:3.0:1769074931.672362:0:40636:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 17 seconds 00000100:00100000:3.0:1769074931.676866:0:40636:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40636:1855007612451968:192.168.204.102@tcp:35:grep.0 00000002:00100000:3.0:1769074931.676899:0:40636:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1006024380 x1855007612451712/t98784247809(98784247809) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1769074931.676959:0:40636:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806fb from it ffff8b100610f9c0 00010000:00010000:3.0:1769074931.676968:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.676980:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.677003:0:40636:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8), inode mode 81a4 mask 44 00000080:00200008:3.0:1769074931.677024:0:40636:0:(file.c:6252:ll_getattr_dentry()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074931.677029:0:40636:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x118:0x0](ffff8b100486c2c8),name=f11.replay-single 00000002:00010000:3.0:1769074931.677048:0:40636:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074931.677061:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.677071:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.677077:0:40636:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074931.677082:0:40636:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) 00000080:00010000:3.0:1769074931.677085:0:40636:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffff8b100486c2c8) for lock 0x68a7113811d806fb 00000080:00010000:3.0:1769074931.677089:0:40636:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806fb from it ffffaa9141067cd8 00010000:00010000:3.0:1769074931.677092:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.677112:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.677131:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074931.677141:0:40636:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:3.0:1769074931.677158:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.677161:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76430, osc lock ffff8b10036171d0, flags 0x400001002 00010000:00010000:3.0:1769074931.677175:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677188:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00010000:3.0:1769074931.677199:0:40636:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 4, setting rss=4; kms=4, end=18446744073709551615 ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.677208:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.677211:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677218:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.677227:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.677236:0:40636:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:3.0:1769074931.677240:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a76ab58 (ffff8b1005685cf8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.677242:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a76ab58 (ffff8b1005685cf8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.677245:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76430 (ffff8b1005685908/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.677249:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a76ab58 (ffff8b1005685cf8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.677252:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76430 (ffff8b1005685cf8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.677260:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1769074931.677272:0:40636:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 0, count: 32768 00000080:00200000:3.0:1769074931.677277:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:3.0:1769074931.677282:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.677286:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074931.677290:0:40636:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:3.0:1769074931.677295:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.677297:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76430, osc lock ffff8b10036171d0, flags 0x400001002 00010000:00010000:3.0:1769074931.677304:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677312:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.677320:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.677323:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677330:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.677338:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.677342:0:40636:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:3.0:1769074931.677344:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.677346:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.677349:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76430 (ffff8b10056853f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.677351:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.677354:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76430 (ffff8b1005685908/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.677359:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.677363:0:40636:0:(file.c:2367:file_read_confine_iter()) f11.replay-single: read [0, 32768] over eof, kms 4, file_size 4. 00000080:00200000:3.0:1769074931.677384:0:40636:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:3.0:1769074931.677396:0:40636:0:(file.c:1994:ll_file_io_generic()) f11.replay-single: read ppos: 0, bytes: 4 00000080:00200000:3.0:1769074931.677401:0:40636:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:3.0:1769074931.677407:0:40636:0:(cl_io.c:214:cl_io_rw_init()) header@000000001b000fd0[0x4, 1, [0x200001b71:0x118:0x0] hash] 00000020:00200000:3.0:1769074931.677409:0:40636:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 4) 0 0 00000080:00200000:3.0:1769074931.677411:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1769074931.677424:0:40636:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4) 4, 4 00020000:00200000:3.0:1769074931.677428:0:40636:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074931.677439:0:40636:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 0] 00000008:00010000:3.0:1769074931.677447:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1769074931.677451:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76430, osc lock ffff8b10036171d0, flags 0x0 00010000:00010000:3.0:1769074931.677459:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677468:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677477:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677484:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.677490:0:40636:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074931.677499:0:40636:0:(vvp_io.c:827:vvp_io_read_start()) f11.replay-single: read [0, 4) 00000080:00200000:3.0:1769074931.677510:0:40636:0:(vvp_io.c:886:vvp_io_read_start()) count:4 iocb pos:0 00000008:00010000:3.0:1769074931.677546:0:40636:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff8b1005666e40 00000008:00010000:3.0:1769074931.677576:0:38151:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8b1005666e40 00010000:00010000:3.0:1769074931.677595:0:38151:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000008:00100000:3.0:1769074931.677629:0:38151:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff8b10037de300 - object 0x240000400:770 offset 0<>4096 00000100:00100000:3.0:1769074931.677692:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612452096:192.168.204.102@tcp:3:grep.0 00000008:00010000:3.0:1769074931.689289:0:2366:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff8b1005666e40 00000020:00200000:3.0:1769074931.689315:0:2366:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1769074931.689332:0:2366:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff8b1005666e40 00000100:00100000:3.0:1769074931.689381:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612452096:192.168.204.102@tcp:3:grep.0 00000020:00010000:0.0:1769074931.690943:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02f38 (ffff8b1005685cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074931.690954:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02f38 (ffff8b1005685cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.690958:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76430 (ffff8b1005685908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074931.690969:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.690982:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074931.690997:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02f38 (ffff8b1005685cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074931.691000:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76430 (ffff8b1005685cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074931.691012:0:40636:0:(vvp_io.c:518:vvp_io_advance()) advancing 4 bytes 00000080:00200000:0.0:1769074931.691025:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074931.691030:0:40636:0:(file.c:2142:ll_file_io_generic()) f11.replay-single: 1 io complete with rc: 0, result: 4, restart: 0 00000080:00200000:0.0:1769074931.691034:0:40636:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 4 00000080:00200008:0.0:1769074931.691126:0:40636:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 4, count: 32768 00000080:00200000:0.0:1769074931.691134:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00000080:00200000:0.0:1769074931.691150:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074931.691156:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074931.691161:0:40636:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:0.0:1769074931.691179:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074931.691183:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d230, osc lock ffff8b10097a2f70, flags 0x400001002 00010000:00010000:0.0:1769074931.691200:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691211:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074931.691221:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074931.691225:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691236:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074931.691245:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074931.691252:0:40636:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:0.0:1769074931.691255:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074931.691257:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.691260:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d230 (ffff8b1007c69e18/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.691262:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f82910 (ffff8b1007c696c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074931.691264:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d230 (ffff8b1007c696c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074931.691271:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074931.691276:0:40636:0:(file.c:2367:file_read_confine_iter()) f11.replay-single: read [4, 32772] over eof, kms 4, file_size 4. 00000080:00200000:0.0:1769074931.691286:0:40636:0:(file.c:1994:ll_file_io_generic()) f11.replay-single: read ppos: 4, bytes: 32768 00000080:00200000:0.0:1769074931.691290:0:40636:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:0.0:1769074931.691297:0:40636:0:(cl_io.c:214:cl_io_rw_init()) header@000000001b000fd0[0x4, 2, [0x200001b71:0x118:0x0] hash] 00000020:00200000:0.0:1769074931.691299:0:40636:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:0.0:1769074931.691301:0:40636:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:0.0:1769074931.691312:0:40636:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:0.0:1769074931.691316:0:40636:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074931.691346:0:40636:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:0.0:1769074931.691357:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074931.691361:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d230, osc lock ffff8b10097a2f70, flags 0x0 00010000:00010000:0.0:1769074931.691370:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691379:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691391:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691397:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691403:0:40636:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 5/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074931.691410:0:40636:0:(vvp_io.c:827:vvp_io_read_start()) f11.replay-single: read [4, 32772) 00000001:00010000:0.0:1769074931.691419:0:40636:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:0.0:1769074931.691424:0:40636:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074931.691426:0:40636:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b100972d7b0, osc lock ffff8b10097a2260, flags 0x400001002 00010000:00010000:0.0:1769074931.691433:0:40636:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691441:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074931.691448:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074931.691451:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 7/1,1 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691458:0:40636:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 6/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074931.691464:0:40636:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074931.691468:0:40636:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 1 00000020:00010000:0.0:1769074931.691471:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_release():245 00000020:00010000:0.0:1769074931.691473:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.691475:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d7b0 (ffff8b1007c696c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.691478:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768248 (ffff8b1005685908/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074931.691480:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d7b0 (ffff8b1005685908/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074931.691485:0:40636:0:(vvp_io.c:886:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:0.0:1769074931.691491:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e02d58 (ffff8b1005685908/0) at cl_lock_release():245 00000020:00010000:0.0:1769074931.691493:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e02d58 (ffff8b1005685908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074931.691495:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100972d230 (ffff8b1007c696c8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074931.691500:0:40636:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 4/1,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074931.691507:0:40636:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 3/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074931.691515:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e02d58 (ffff8b1005685908/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074931.691517:0:40636:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100972d230 (ffff8b1005685908/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074931.691521:0:40636:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000080:00200000:0.0:1769074931.691527:0:40636:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074931.691530:0:40636:0:(file.c:2142:ll_file_io_generic()) f11.replay-single: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:0.0:1769074931.691533:0:40636:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:0.0:1769074931.691548:0:40636:0:(file.c:424:ll_file_release()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffff8b100486c2c8), flags 100000 00010000:00010000:0.0:1769074931.691568:0:40636:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064b6e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074931.705272:0:40641:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074931.705300:0:40641:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074931.705317:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074931.705333:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074931.705340:0:40641:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074931.705348:0:40641:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074931.705351:0:40641:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:2.0:1769074931.705358:0:40641:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa9140d9bbf8 00010000:00010000:2.0:1769074931.705361:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074931.705369:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074931.705384:0:40641:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074931.705398:0:40641:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b101082b500, open_flags 8100, mode 0 opened 0 00000080:00200000:2.0:1769074931.705405:0:40641:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:2.0:1769074931.705420:0:40641:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f11.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:2.0:1769074931.705476:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 40641 initiator: MDT0 00010000:00010000:2.0:1769074931.705483:0:40641:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40641 initiator: MDT0 00000020:00100000:2.0:1769074931.705957:0:40641:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074931.705971:0:40641:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d5ad80 x1855007612452352/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074931.708165:0:40641:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40641 initiator: MDT0 00000100:00100000:2.0:1769074931.708213:0:40641:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40641:1855007612452352:192.168.204.102@tcp:101:grep.0 00000100:00100000:2.0:1769074931.708603:0:40641:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd240 going to sleep for 11 seconds 00000100:00100000:3.0:1769074931.743338:0:40641:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40641:1855007612452352:192.168.204.102@tcp:101:grep.0 00010000:00010000:3.0:1769074931.743396:0:40641:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743426:0:40641:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743436:0:40641:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743450:0:40641:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743474:0:40641:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743489:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743497:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074931.743505:0:40641:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074931.743523:0:40641:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff8b1005d5ad80 x1855007612452352/t98784247811(98784247811) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074931.743539:0:40641:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5ad80 x1855007612452352/t98784247811(98784247811) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074931.743553:0:40641:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 0 00000002:00010000:3.0:1769074931.743568:0:40641:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.743580:0:40641:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:3.0:1769074931.743587:0:40641:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074931.743668:0:40641:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffff8b1004859148): rc = 0 00000080:00200000:3.0:1769074931.743698:0:40641:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:3.0:1769074931.743706:0:40641:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d80702 00000080:00010000:3.0:1769074931.743720:0:40641:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00200008:3.0:1769074931.743740:0:40641:0:(file.c:1001:ll_file_open()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 100400 00000002:00100000:3.0:1769074931.743769:0:40641:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b1005d5ad80 x1855007612452352/t98784247811(98784247811) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1769074931.743828:0:40641:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80702 from it ffff8b1005664180 00010000:00010000:3.0:1769074931.743833:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.743841:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.743887:0:40641:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148), inode mode 81a4 mask 44 00000080:00200008:3.0:1769074931.743918:0:40641:0:(file.c:6252:ll_getattr_dentry()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074931.743926:0:40641:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x115:0x0](ffff8b1004859148),name=f11.replay-single-2 00000002:00010000:3.0:1769074931.743951:0:40641:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x115:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074931.743961:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.743974:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.743982:0:40641:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074931.743988:0:40641:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) 00000080:00010000:3.0:1769074931.743992:0:40641:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffff8b1004859148) for lock 0x68a7113811d80702 00000080:00010000:3.0:1769074931.743996:0:40641:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80702 from it ffffaa9140d9bcd8 00010000:00010000:3.0:1769074931.743999:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074931.744006:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00000080:00200000:3.0:1769074931.744025:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074931.744038:0:40641:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:3.0:1769074931.744061:0:40641:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.744066:0:40641:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76cb0, osc lock ffff8b1003616980, flags 0x400001002 00010000:00010000:3.0:1769074931.744334:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744349:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744363:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.744368:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744376:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744388:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.744405:0:40641:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 3 00000020:00010000:3.0:1769074931.744410:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.744413:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744416:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76cb0 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744421:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.744423:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76cb0 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.744436:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1769074931.744456:0:40641:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 0, count: 32768 00000080:00200000:3.0:1769074931.744475:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074931.744483:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.744487:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074931.744491:0:40641:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:3.0:1769074931.744537:0:40641:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.744539:0:40641:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76cb0, osc lock ffff8b1003616980, flags 0x400001002 00010000:00010000:3.0:1769074931.744549:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744559:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744567:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.744571:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744579:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744589:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.744594:0:40641:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 3 00000020:00010000:3.0:1769074931.744597:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.744600:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744603:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76cb0 (ffff8b1005685998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744606:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.744608:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76cb0 (ffff8b1005685878/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.744615:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.744620:0:40641:0:(file.c:2367:file_read_confine_iter()) f11.replay-single-2: read [0, 32768] over eof, kms 4, file_size 4. 00000080:00200008:3.0:1769074931.744869:0:40641:0:(file.c:2474:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 4, count: 32768 00000080:00200000:3.0:1769074931.744874:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074931.744880:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.744883:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1769074931.744886:0:40641:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:3.0:1769074931.744894:0:40641:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.744896:0:40641:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76cb0, osc lock ffff8b1003616980, flags 0x400001002 00010000:00010000:3.0:1769074931.744906:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744915:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744922:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.744925:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.744932:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.744940:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.744944:0:40641:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 3 00000020:00010000:3.0:1769074931.744946:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.744948:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744951:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76cb0 (ffff8b1005685998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.744954:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a7686d8 (ffff8b1005685878/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.744956:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76cb0 (ffff8b1005685878/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.744962:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.744966:0:40641:0:(file.c:2367:file_read_confine_iter()) f11.replay-single-2: read [4, 32772] over eof, kms 4, file_size 4. 00000080:00200000:3.0:1769074931.744972:0:40641:0:(file.c:1994:ll_file_io_generic()) f11.replay-single-2: read ppos: 4, bytes: 32768 00000080:00200000:3.0:1769074931.744976:0:40641:0:(file.c:1759:ll_io_set_mirror()) f11.replay-single-2: desiginated mirror: 0 00000020:00200000:3.0:1769074931.744984:0:40641:0:(cl_io.c:214:cl_io_rw_init()) header@000000007469e5a4[0x4, 2, [0x200001b71:0x115:0x0] hash] 00000020:00200000:3.0:1769074931.744986:0:40641:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:3.0:1769074931.744989:0:40641:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1769074931.745000:0:40641:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:3.0:1769074931.745005:0:40641:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074931.745018:0:40641:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:3.0:1769074931.745027:0:40641:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1769074931.745031:0:40641:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76cb0, osc lock ffff8b1003616980, flags 0x0 00010000:00010000:3.0:1769074931.745041:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745051:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745060:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745067:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745074:0:40641:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 5/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000080:00200000:3.0:1769074931.745269:0:40641:0:(vvp_io.c:827:vvp_io_read_start()) f11.replay-single-2: read [4, 32772) 00000001:00010000:3.0:1769074931.745283:0:40641:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:3.0:1769074931.745290:0:40641:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1769074931.745292:0:40641:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff8b1007f76a30, osc lock ffff8b1003617c80, flags 0x400001002 00010000:00010000:3.0:1769074931.745299:0:40641:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745307:0:40641:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.745322:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1769074931.745325:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 7/1,1 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.745332:0:40641:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 6/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00200000:3.0:1769074931.745338:0:40641:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1769074931.745341:0:40641:0:(file.c:1686:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 3 00000020:00010000:3.0:1769074931.745343:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.745345:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.745347:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76a30 (ffff8b1005685878/1) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.745349:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b100a768b68 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.745352:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76a30 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.746792:0:40641:0:(vvp_io.c:886:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:3.0:1769074931.746817:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8b1007e026c8 (ffff8b10056853f8/0) at cl_lock_release():245 00000020:00010000:3.0:1769074931.746822:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007e026c8 (ffff8b10056853f8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1769074931.746825:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8b1007f76cb0 (ffff8b1005685878/1) at cl_lock_cancel():142 00010000:00010000:3.0:1769074931.746834:0:40641:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 4/1,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074931.746846:0:40641:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00000020:00010000:3.0:1769074931.746864:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007e026c8 (ffff8b10056853f8/0) at cl_lock_fini():72 00000020:00010000:3.0:1769074931.746867:0:40641:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8b1007f76cb0 (ffff8b10056853f8/1) at cl_lock_fini():72 00000080:00200000:3.0:1769074931.746877:0:40641:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000080:00200000:3.0:1769074931.746891:0:40641:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074931.746896:0:40641:0:(file.c:2142:ll_file_io_generic()) f11.replay-single-2: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:3.0:1769074931.746901:0:40641:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:3.0:1769074931.746927:0:40641:0:(file.c:424:ll_file_release()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffff8b1004859148), flags 100000 00000002:00100000:3.0:1769074931.747013:0:40641:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005d5ad80 x1855007612452352/t98784247811(98784247811) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1769074931.747029:0:40641:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1769074931.747036:0:40641:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de300 x1855007612452608/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1769074931.747062:0:40641:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40641:1855007612452608:192.168.204.102@tcp:35:grep.0 00000100:00100000:3.0:1769074931.747755:0:40641:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c780 going to sleep for 11 seconds 00000100:00100000:3.0:1769074931.765549:0:40641:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job grep:93598d0a-9484-4e40-94b2-393ee542cbad:40641:1855007612452608:192.168.204.102@tcp:35:grep.0 00000002:00100000:3.0:1769074931.765597:0:40641:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b1005d5ad80 x1855007612452352/t98784247811(98784247811) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 592/664 e 0 to 0 dl 1769074947 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1769074932.256973:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074933.283200:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074934.304171:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074935.327783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074936.353494:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074936.484995:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612452736:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.485148:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612452992:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.485192:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074936.485200:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612452864:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.485226:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612453120:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.501692:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612452864:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.501718:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612453120:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.501733:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612452736:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.501746:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612452992:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074937.380110:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074938.399935:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1769074938.476601:0:41203:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 12: open, unlink |X| close ========= 04:42:17 (1769074937) 00000080:00200000:1.0:1769074938.790370:0:41215:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074938.790413:0:41215:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074938.790436:0:41215:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.790455:0:41215:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.790464:0:41215:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074938.790476:0:41215:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074938.790480:0:41215:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074938.790489:0:41215:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa9140d7fbd0 00010000:00010000:1.0:1769074938.790493:0:41215:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.790503:0:41215:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.790524:0:41215:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:1.0:1769074938.790924:0:41215:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:1.0:1769074938.790931:0:41215:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074938.790941:0:41215:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074938.790949:0:41215:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.790959:0:41215:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.790967:0:41215:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074938.790972:0:41215:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074938.790975:0:41215:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074938.791089:0:41215:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa9140d7fcd0 00010000:00010000:1.0:1769074938.791092:0:41215:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.791100:0:41215:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.791111:0:41215:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074938.791119:0:41215:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074938.791129:0:41215:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074938.791137:0:41215:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.791146:0:41215:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.791151:0:41215:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074938.791155:0:41215:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:1.0:1769074938.791157:0:41215:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d806f4 00000080:00010000:1.0:1769074938.791161:0:41215:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d806f4 from it ffffaa9140d7fdb0 00010000:00010000:1.0:1769074938.791164:0:41215:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.791172:0:41215:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074938.791179:0:41215:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:1.0:1769074938.791185:0:41215:0:(namei.c:2065:ll_create_nd()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:1.0:1769074938.791189:0:41215:0:(namei.c:2020:ll_mknod()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:1.0:1769074938.791221:0:41215:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.791290:0:41215:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074938.791298:0:41215:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074938.791301:0:41215:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d806f4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064d51 expref: -99 pid: 40178 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074938.791333:0:41215:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074938.791350:0:41215:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006136a00 x1855007612453376/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074938.791381:0:41215:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:41215:1855007612453376:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:1.0:1769074938.791547:0:41215:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56e40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074938.809144:0:41215:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247812 gen 1 00000100:00100000:2.0:1769074938.809168:0:41215:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006136a00 x1855007612453376/t98784247813(98784247813) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/448 e 0 to 0 dl 1769074954 ref 3 fl Rpc:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074938.809184:0:41215:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:41215:1855007612453376:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:2.0:1769074938.809246:0:41215:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x119:0x0](ffff8b100486e3c8) 00000080:00200000:2.0:1769074938.809296:0:41215:0:(namei.c:2073:ll_create_nd()) VFS Op:name=f12.replay-single, unhashed 1 00000080:00200000:3.0:1769074938.840080:0:41217:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074938.840109:0:41217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074938.840173:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 41217 initiator: MDT0 00010000:00010000:3.0:1769074938.840184:0:41217:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00010000:00010000:3.0:1769074938.840194:0:41217:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00000100:00100000:3.0:1769074938.840220:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612453504:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074938.840311:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074938.849140:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612453504:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074938.849179:0:41217:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849195:0:41217:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849209:0:41217:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849222:0:41217:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849235:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849243:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849251:0:41217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074938.849262:0:41217:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b10037de300 x1855007612453504/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074938.849277:0:41217:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849293:0:41217:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1769074938.849300:0:41217:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1769074938.849337:0:41217:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:3.0:1769074938.849345:0:41217:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80709 00000080:00010000:3.0:1769074938.849352:0:41217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074938.849356:0:41217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80709 from it ffffaa9141cfbbf8 00010000:00010000:3.0:1769074938.849359:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.849368:0:41217:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.849400:0:41217:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074938.849428:0:41217:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), file ffff8b1005eebe00, open_flags 8000, mode 0 opened 0 00000080:00200000:3.0:1769074938.849436:0:41217:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=open 00000002:00010000:3.0:1769074938.849460:0:41217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f12.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:3.0:1769074938.849521:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 41217 initiator: MDT0 00010000:00010000:3.0:1769074938.849528:0:41217:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00000020:00100000:3.0:1769074938.849543:0:41217:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074938.849550:0:41217:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037de300 x1855007612453760/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074938.849559:0:41217:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00000100:00100000:3.0:1769074938.849587:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612453760:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074938.849927:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074938.853226:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612453760:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074938.853262:0:41217:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853289:0:41217:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853297:0:41217:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853309:0:41217:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853321:0:41217:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853335:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853343:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853351:0:41217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074938.853363:0:41217:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff8b10037de300 x1855007612453760/t98784247814(98784247814) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074954 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1769074938.853379:0:41217:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037de300 x1855007612453760/t98784247814(98784247814) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074954 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074938.853391:0:41217:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853403:0:41217:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/281 (0 0) 00000080:00200000:3.0:1769074938.853411:0:41217:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074938.853437:0:41217:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x119:0x0], updating i_size 0 00000080:00200000:3.0:1769074938.853440:0:41217:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x119:0x0](ffff8b100486e3c8): rc = 0 00000080:00200000:3.0:1769074938.853461:0:41217:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x119:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074938.853467:0:41217:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) for lock 0x68a7113811d80710 00000080:00010000:3.0:1769074938.853481:0:41217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) 00000080:00200008:3.0:1769074938.853501:0:41217:0:(file.c:1001:ll_file_open()) START file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8), flags 100000 00000002:00100000:3.0:1769074938.853527:0:41217:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff8b10037de300 x1855007612453760/t98784247814(98784247814) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074954 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1769074938.853617:0:41217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80710 from it ffff8b10065d7360 00010000:00010000:3.0:1769074938.853622:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.853631:0:41217:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.853708:0:41217:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x119:0x0](ffff8b100486e3c8), inode mode 81a4 mask 44 00000080:00200000:3.0:1769074938.878182:0:41219:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074938.878221:0:41219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074938.878244:0:41219:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878265:0:41219:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878273:0:41219:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074938.878282:0:41219:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074938.878287:0:41219:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80709 00000080:00010000:3.0:1769074938.878295:0:41219:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80709 from it ffffaa91415bbb58 00010000:00010000:3.0:1769074938.878300:0:41219:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878309:0:41219:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878331:0:41219:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074938.878341:0:41219:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f12.replay-single, flags=8 00000080:00200008:3.0:1769074938.878349:0:41219:0:(file.c:6252:ll_getattr_dentry()) START file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074938.878354:0:41219:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x119:0x0](ffff8b100486e3c8),name=f12.replay-single 00000002:00010000:3.0:1769074938.878367:0:41219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x119:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074938.878374:0:41219:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878383:0:41219:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878390:0:41219:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074938.878394:0:41219:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) 00000080:00010000:3.0:1769074938.878397:0:41219:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) for lock 0x68a7113811d80710 00000080:00010000:3.0:1769074938.878401:0:41219:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80710 from it ffffaa91415bbcb8 00010000:00010000:3.0:1769074938.878404:0:41219:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878412:0:41219:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878439:0:41219:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074938.878445:0:41219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074938.878450:0:41219:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878459:0:41219:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878465:0:41219:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074938.878470:0:41219:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074938.878472:0:41219:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80709 00000080:00010000:3.0:1769074938.878475:0:41219:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80709 from it ffffaa91415bbc10 00010000:00010000:3.0:1769074938.878479:0:41219:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878486:0:41219:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878493:0:41219:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074938.878500:0:41219:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f12.replay-single, flags=0 00000080:00200000:3.0:1769074938.878504:0:41219:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074938.878509:0:41219:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074938.878514:0:41219:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878521:0:41219:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878528:0:41219:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074938.878543:0:41219:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074938.878546:0:41219:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80709 00000080:00010000:3.0:1769074938.878549:0:41219:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80709 from it ffffaa91415bbd68 00010000:00010000:3.0:1769074938.878551:0:41219:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878558:0:41219:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878566:0:41219:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074938.878573:0:41219:0:(namei.c:2354:ll_unlink()) VFS Op:name=f12.replay-single, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074938.878615:0:41219:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878678:0:41219:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074938.878690:0:41219:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x119:0x0]: losing layout lock 00010000:00010000:3.0:1769074938.878741:0:41219:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878749:0:41219:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878759:0:41219:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074938.878762:0:41219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169000/0x68a7113811d80710 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064ddd expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074938.878780:0:41219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d80709 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064dc1 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074938.878802:0:41219:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074938.878818:0:41219:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037ddf80 x1855007612454016/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074938.878848:0:41219:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41219:1855007612454016:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074938.878965:0:41219:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074938.887337:0:41219:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41219:1855007612454016:192.168.204.102@tcp:36:rm.0 00010000:00010000:3.0:1769074939.423975:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1769074939.684822:0:41229:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074939.684843:0:41229:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:1.0:1769074939.684872:0:41229:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074939.684959:0:41229:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 41229 initiator: MDT0 00010000:00010000:1.0:1769074939.684981:0:41229:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41229 initiator: MDT0 00010000:00010000:1.0:1769074939.684994:0:41229:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41229 initiator: MDT0 00000100:00100000:1.0:1769074939.685029:0:41229:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454144:192.168.204.102@tcp:101:lfs.0 00000100:00100000:1.0:1769074939.685146:0:41229:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56b40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074939.704063:0:41229:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454144:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074939.704101:0:41229:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704118:0:41229:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704132:0:41229:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704144:0:41229:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704163:0:41229:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704172:0:41229:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704181:0:41229:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074939.704194:0:41229:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006135880 x1855007612454144/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074955 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074939.704209:0:41229:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704224:0:41229:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074939.704231:0:41229:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074939.704281:0:41229:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074939.704289:0:41229:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.704296:0:41229:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.704300:0:41229:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9140ddbcb8 00010000:00010000:2.0:1769074939.704303:0:41229:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.704312:0:41229:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074939.705236:0:41229:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074939.705248:0:41229:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074939.705271:0:41229:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.705284:0:41229:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.705298:0:41229:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.705306:0:41229:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074939.705311:0:41229:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.705314:0:41229:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.705320:0:41229:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9140ddbcb8 00010000:00010000:2.0:1769074939.705323:0:41229:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.705330:0:41229:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.705449:0:41229:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074939.705458:0:41229:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.705465:0:41229:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.705474:0:41229:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.705480:0:41229:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074939.705484:0:41229:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.705486:0:41229:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.705491:0:41229:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9140ddbbb0 00010000:00010000:2.0:1769074939.705494:0:41229:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.705501:0:41229:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.705508:0:41229:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074939.705517:0:41229:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074939.705623:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.705663:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00800000:00000004:2.0:1769074939.705672:0:41229:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1924, max_age 1931 00000100:00100000:2.0:1769074939.705727:0:41229:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454272:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074939.705844:0:41229:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074939.712005:0:41229:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454272:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074939.712046:0:41229:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 500913/501262 00000080:00200080:2.0:1769074939.712147:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712174:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712184:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712188:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712193:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712196:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712200:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712204:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712208:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712212:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712217:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712220:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712224:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712227:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712231:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712235:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712239:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712243:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712247:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712250:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712254:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712258:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712276:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712281:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712286:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712290:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712294:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712298:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712302:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712306:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712310:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712314:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712319:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712323:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712328:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712332:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712337:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712341:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712346:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712350:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712354:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712358:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712362:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712366:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712370:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712374:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712377:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712381:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712384:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712388:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712392:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712395:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712399:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712403:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712406:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712410:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712413:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712427:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712431:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712435:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712439:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712443:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712447:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712450:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712455:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712458:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712462:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712466:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712470:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712474:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712478:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712481:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712486:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712489:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712494:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712497:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712501:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712505:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712509:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712512:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712516:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712520:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712524:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712527:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712531:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712535:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712538:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712541:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712545:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712548:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712552:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712555:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712559:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712562:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712566:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712570:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712574:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712577:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712581:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712585:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712589:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712594:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712597:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712601:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712604:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.712608:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.712612:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720634:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720698:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720711:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720718:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720723:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720727:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720731:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720736:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720739:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720744:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720757:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720763:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720768:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720772:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720775:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720780:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720783:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720787:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720790:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720794:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00800000:00000080:2.0:1769074939.720798:0:41229:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.720802:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.720809:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00020000:00000004:2.0:1769074939.720818:0:41229:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1924, max_age 1931 00000100:00100000:2.0:1769074939.720884:0:41229:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454400:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074939.721877:0:41229:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074939.738598:0:41229:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454400:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074939.738679:0:41229:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:2.0:1769074939.739052:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.739075:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00020000:00000004:2.0:1769074939.739084:0:41229:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1924, max_age 1931 00000100:00100000:2.0:1769074939.739132:0:41229:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454528:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074939.739224:0:41229:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd000 going to sleep for 11 seconds 00000100:00100000:2.0:1769074939.741351:0:41229:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41229:1855007612454528:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074939.741386:0:41229:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:2.0:1769074939.741547:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741566:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741576:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741580:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741584:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741588:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741592:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741596:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741600:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741604:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741607:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741611:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741839:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741848:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741856:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741860:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741864:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741868:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741872:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741876:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741879:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741883:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741887:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741891:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741895:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741898:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741902:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741905:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741908:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741911:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741915:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741917:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741921:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741924:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741927:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741929:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.741933:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.741936:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742186:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742192:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742197:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742201:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742205:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742208:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742212:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742215:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742220:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742223:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742227:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742230:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742235:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742238:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742242:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742245:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742249:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742252:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742256:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742259:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742344:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742350:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742354:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742357:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742361:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742364:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742368:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742371:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742375:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742379:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742383:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742387:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742390:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742395:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742399:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742402:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742407:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742410:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742413:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742513:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742521:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742525:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742529:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742532:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742535:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742538:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742541:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742544:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742548:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742551:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742555:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742558:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742562:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742566:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742570:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742574:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742578:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742582:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742586:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742590:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742593:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742597:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742600:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742603:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742607:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.742610:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.742613:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743413:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743531:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743539:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743543:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743547:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743552:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743555:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743560:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743563:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743567:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743570:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743574:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743578:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743582:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743586:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743590:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743594:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743598:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743602:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200080:2.0:1769074939.743606:0:41229:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7ffe92743750 00020000:00000080:2.0:1769074939.743610:0:41229:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aa800 uarg=0000000000000000 00000080:00200008:2.0:1769074939.745187:0:41229:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200000:2.0:1769074939.832127:0:41236:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074939.832171:0:41236:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.832195:0:41236:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.832216:0:41236:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.832225:0:41236:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074939.832239:0:41236:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.832244:0:41236:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.832253:0:41236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9141bb7bd0 00010000:00010000:2.0:1769074939.832257:0:41236:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.832267:0:41236:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.832293:0:41236:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074939.832316:0:41236:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=1536 00000080:00200000:2.0:1769074939.832319:0:41236:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074939.832325:0:41236:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.832331:0:41236:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.832339:0:41236:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.832345:0:41236:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074939.832349:0:41236:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.832352:0:41236:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.832355:0:41236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9141bb7cd0 00010000:00010000:2.0:1769074939.832358:0:41236:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.832365:0:41236:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.832372:0:41236:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074939.833634:0:41236:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074939.833871:0:41236:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.833887:0:41236:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.833905:0:41236:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.833915:0:41236:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074939.833922:0:41236:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.833927:0:41236:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d80717 00000080:00010000:2.0:1769074939.833933:0:41236:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80717 from it ffffaa9141bb7db0 00010000:00010000:2.0:1769074939.833947:0:41236:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.833956:0:41236:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.833981:0:41236:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:2.0:1769074939.833990:0:41236:0:(namei.c:2065:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=33188, excl=1 00000080:00200000:2.0:1769074939.833994:0:41236:0:(namei.c:2020:ll_mknod()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) mode 100644 dev 0 00010000:00010000:2.0:1769074939.834028:0:41236:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.834103:0:41236:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.834111:0:41236:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074939.834115:0:41236:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10060b8e00/0x68a7113811d80717 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064e07 expref: -99 pid: 41229 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074939.834151:0:41236:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074939.834167:0:41236:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d58380 x1855007612454784/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074939.834198:0:41236:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:41236:1855007612454784:192.168.204.102@tcp:36:mcreate.0 00000100:00100000:2.0:1769074939.835597:0:41236:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd000 going to sleep for 11 seconds 00000100:00100000:3.0:1769074939.866490:0:41236:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job mcreate:93598d0a-9484-4e40-94b2-393ee542cbad:41236:1855007612454784:192.168.204.102@tcp:36:mcreate.0 00000080:00200000:3.0:1769074939.867115:0:41236:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x11a:0x0](ffff8b1004858908) 00000080:00200000:3.0:1769074939.867171:0:41236:0:(namei.c:2073:ll_create_nd()) VFS Op:name=fsa-oleg402-client.virtnet, unhashed 1 00000080:00200000:0.0:1769074939.897700:0:41239:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074939.897741:0:41239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074939.897826:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 41239 initiator: MDT0 00010000:00010000:0.0:1769074939.897836:0:41239:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41239 initiator: MDT0 00010000:00010000:0.0:1769074939.897845:0:41239:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41239 initiator: MDT0 00000100:00100000:0.0:1769074939.897882:0:41239:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612454912:192.168.204.102@tcp:101:rm.0 00000100:00100000:0.0:1769074939.897992:0:41239:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a0300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074939.905794:0:41239:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612454912:192.168.204.102@tcp:101:rm.0 00010000:00010000:2.0:1769074939.905834:0:41239:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905849:0:41239:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905862:0:41239:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905875:0:41239:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905888:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905896:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.905904:0:41239:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074939.905916:0:41239:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024700 x1855007612454912/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074955 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074939.905931:0:41239:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.906029:0:41239:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074939.906038:0:41239:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074939.906078:0:41239:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074939.906085:0:41239:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8071e 00000080:00010000:2.0:1769074939.906093:0:41239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074939.906096:0:41239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8071e from it ffffaa9141067b58 00010000:00010000:2.0:1769074939.906100:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074939.906108:0:41239:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074939.906141:0:41239:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:2.0:1769074939.906167:0:41239:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), flags=8 00000080:00200000:2.0:1769074939.906173:0:41239:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08), intent=getattr 00000002:00010000:2.0:1769074939.906202:0:41239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: fsa-oleg402-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074939.906257:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 41239 initiator: MDT0 00010000:00010000:2.0:1769074939.906264:0:41239:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41239 initiator: MDT0 00010000:00010000:2.0:1769074939.906273:0:41239:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41239 initiator: MDT0 00000100:00100000:2.0:1769074939.906297:0:41239:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612455040:192.168.204.102@tcp:101:rm.0 00000100:00100000:2.0:1769074939.906398:0:41239:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd6c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074939.919683:0:41239:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612455040:192.168.204.102@tcp:101:rm.0 00010000:00010000:3.0:1769074939.919721:0:41239:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919751:0:41239:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919761:0:41239:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919775:0:41239:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919786:0:41239:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919798:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919806:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919813:0:41239:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000002:00100000:3.0:1769074939.919826:0:41239:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59f80 x1855007612455040/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 600/584 e 0 to 0 dl 1769074955 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074939.919839:0:41239:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919850:0:41239:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/282 (0 0) 00000080:00200000:3.0:1769074939.919858:0:41239:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1769074939.919899:0:41239:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x11a:0x0], updating i_size 0 00000080:00200000:3.0:1769074939.919903:0:41239:0:(namei.c:151:ll_iget()) got inode: [0x200001b71:0x11a:0x0](ffff8b1004858908): rc = 0 00000080:00200000:3.0:1769074939.919924:0:41239:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x11a:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1769074939.919930:0:41239:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x11a:0x0](ffff8b1004858908) for lock 0x68a7113811d80725 00000080:00010000:3.0:1769074939.919944:0:41239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x11a:0x0](ffff8b1004858908) 00000080:00010000:3.0:1769074939.919949:0:41239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80725 from it ffffaa9141067b20 00010000:00010000:3.0:1769074939.919953:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.919972:0:41239:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074939.920025:0:41239:0:(file.c:6252:ll_getattr_dentry()) START file fsa-oleg402-client.virtnet:[0x200001b71:0x11a:0x0](ffff8b1004858908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1769074939.920033:0:41239:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x11a:0x0](ffff8b1004858908),name=fsa-oleg402-client.virtnet 00000002:00010000:3.0:1769074939.920059:0:41239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x11a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074939.920069:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920081:0:41239:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920087:0:41239:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074939.920093:0:41239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x11a:0x0](ffff8b1004858908) 00000080:00010000:3.0:1769074939.920109:0:41239:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x11a:0x0](ffff8b1004858908) for lock 0x68a7113811d80725 00000080:00010000:3.0:1769074939.920113:0:41239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80725 from it ffffaa9141067cb8 00010000:00010000:3.0:1769074939.920116:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920123:0:41239:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920195:0:41239:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074939.920203:0:41239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074939.920210:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920218:0:41239:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920224:0:41239:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074939.920227:0:41239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074939.920230:0:41239:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8071e 00000080:00010000:3.0:1769074939.920234:0:41239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8071e from it ffffaa9141067c10 00010000:00010000:3.0:1769074939.920237:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920244:0:41239:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920251:0:41239:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 1 00000080:00200000:3.0:1769074939.920263:0:41239:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg402-client.virtnet, flags=0 00000080:00200000:3.0:1769074939.920269:0:41239:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:3.0:1769074939.920275:0:41239:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074939.920281:0:41239:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920289:0:41239:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920295:0:41239:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074939.920299:0:41239:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:3.0:1769074939.920302:0:41239:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8071e 00000080:00010000:3.0:1769074939.920305:0:41239:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8071e from it ffffaa9141067d68 00010000:00010000:3.0:1769074939.920308:0:41239:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920316:0:41239:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920324:0:41239:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 3 00000080:00200000:3.0:1769074939.920330:0:41239:0:(namei.c:2354:ll_unlink()) VFS Op:name=fsa-oleg402-client.virtnet, dir=[0x200000007:0x1:0x0](ffff8b1018194b08) 00010000:00010000:3.0:1769074939.920353:0:41239:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920393:0:41239:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074939.920404:0:41239:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x11a:0x0]: losing layout lock 00010000:00010000:3.0:1769074939.920448:0:41239:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920456:0:41239:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920461:0:41239:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1769074939.920464:0:41239:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80725 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x11a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x143201ebf3064e3f expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074939.920483:0:41239:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8071e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x143201ebf3064e23 expref: -99 pid: 41239 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074939.920504:0:41239:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074939.920513:0:41239:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b10037dc000 x1855007612455296/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074939.920551:0:41239:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612455296:192.168.204.102@tcp:36:rm.0 00000100:00100000:3.0:1769074939.920687:0:41239:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074939.931104:0:41239:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job rm:93598d0a-9484-4e40-94b2-393ee542cbad:41239:1855007612455296:192.168.204.102@tcp:36:rm.0 00000080:00200000:3.0:1769074939.931195:0:41239:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x11a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769074939.931211:0:41239:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x11a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1769074939.931220:0:41239:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x11a:0x0](ffff8b1004858908) 00010000:00010000:0.0:1769074940.449774:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1769074941.089662:0:8:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1769074941.089685:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074941.089722:0:8:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1769074941.472117:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074941.600438:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612455552:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.600688:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612455424:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.601138:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612455808:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.606212:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612455680:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.620326:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612455424:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.620368:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612455680:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.620769:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612455552:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.620784:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612455808:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074942.501411:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1769074943.192354:0:41269:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200008:1.0:1769074943.201422:0:41217:0:(llite_lib.c:2314:ll_setattr_raw()) START file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8) current size 0, valid attrs 41, mode 8000, uid 1, gid 0, new size 18446744071721085669, atime 1769074943.000000000, mtime 1769074943.000000000, ctime 1769074943.000000000 00000020:00100000:1.0:1769074943.201758:0:41217:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074943.201775:0:41217:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1006137800 x1855007612456064/t0(0) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074943.201809:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456064:192.168.204.102@tcp:36:multiop.0 00000100:00100000:1.0:1769074943.202024:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c560c0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074943.221862:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456064:192.168.204.102@tcp:36:multiop.0 00000080:00200000:3.0:1769074943.221954:0:41217:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x119:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:3.0:1769074943.221997:0:41217:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200001b71:0x119:0x0](ffff8b100486e3c8) 00000002:00010000:3.0:1769074943.222010:0:41217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x119:0x0]) in obj [0x200001b71:0x119:0x0], intent: layout flags 0 00010000:00010000:3.0:1769074943.222080:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 41217 initiator: MDT0 00010000:00010000:3.0:1769074943.222091:0:41217:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00010000:00010000:3.0:1769074943.222104:0:41217:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00000100:00100000:3.0:1769074943.222128:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456192:192.168.204.102@tcp:101:multiop.0 00000100:00100000:3.0:1769074943.223539:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:3.0:1769074943.237486:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456192:192.168.204.102@tcp:101:multiop.0 00010000:00010000:3.0:1769074943.237541:0:41217:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074943.237561:0:41217:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00000002:00100000:3.0:1769074943.237585:0:41217:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff8b10037dd500 x1855007612456192/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 376/344 e 0 to 0 dl 1769074959 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1769074943.237598:0:41217:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074943.237613:0:41217:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 16 res 8589941617/281 (0 0) 00000080:00200000:3.0:1769074943.237621:0:41217:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200001b71:0x119:0x0] suppgids 0 0: rc 0 00000080:00010000:3.0:1769074943.237695:0:41217:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) for lock 0x68a7113811d8072c 00000080:00010000:3.0:1769074943.237707:0:41217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8072c from it ffffaa9141cfbaf8 00010000:00010000:3.0:1769074943.237711:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074943.237721:0:41217:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074943.237753:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 2/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:3.0:1769074943.237766:0:41217:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 2/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00000080:00010000:3.0:1769074943.237775:0:41217:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200001b71:0x119:0x0](000000000f9298bc) being reconfigured ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00000100:00100000:3.0:1769074943.237840:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456320:192.168.204.102@tcp:49:multiop.0 00000100:00100000:2.0:1769074943.238396:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10062a2300 going to sleep for 11 seconds 00000100:00100000:2.0:1769074943.241120:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456320:192.168.204.102@tcp:49:multiop.0 00000080:00200000:2.0:1769074943.241175:0:41217:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x119:0x0]: layout version change: 4294967294 -> 4294967295 00010000:00010000:2.0:1769074943.241188:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074943.241201:0:41217:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074943.241230:0:41217:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x119:0x0] ignore/verify layout 0/0, layout version -1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074943.241247:0:41217:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8) current size 0, valid attrs 41, mode 8000, uid 1, gid 0, new size 18446744071721085669, atime 1769074943.000000000, mtime 1769074943.000000000, ctime 1769074943.000000000, rc 0 00000080:00200008:2.0:1769074943.241288:0:41217:0:(file.c:6252:ll_getattr_dentry()) START file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074943.241298:0:41217:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x119:0x0](ffff8b100486e3c8),name=f12.replay-single 00000002:00010000:2.0:1769074943.241320:0:41217:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200001b71:0x119:0x0]) in obj [0x200001b71:0x119:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074943.241334:0:41217:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 30 res 8589941617/281 (0 0) 00010000:00010000:2.0:1769074943.241391:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 41217 initiator: MDT0 00010000:00010000:2.0:1769074943.241399:0:41217:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00010000:00010000:2.0:1769074943.241410:0:41217:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 41217 initiator: MDT0 00000100:00100000:2.0:1769074943.241439:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456448:192.168.204.102@tcp:101:multiop.0 00000100:00100000:2.0:1769074943.241534:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074943.250694:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456448:192.168.204.102@tcp:101:multiop.0 00010000:00010000:2.0:1769074943.250733:0:41217:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250748:0:41217:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250762:0:41217:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250775:0:41217:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250789:0:41217:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250799:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250807:0:41217:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074943.250820:0:41217:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1005d59f80 x1855007612456448/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/584 e 0 to 0 dl 1769074959 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074943.250836:0:41217:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250853:0:41217:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589941617/281 (0 0) 00000080:00200000:2.0:1769074943.250861:0:41217:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x119:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074943.250888:0:41217:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200001b71:0x119:0x0], updating i_size 0 00000080:00200000:2.0:1769074943.250916:0:41217:0:(file.c:6809:ll_layout_conf()) [0x200001b71:0x119:0x0]: layout version change: 4294967295 -> 4294967295 00000080:00010000:2.0:1769074943.250930:0:41217:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) for lock 0x68a7113811d80733 00000080:00010000:2.0:1769074943.250937:0:41217:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x119:0x0](ffff8b100486e3c8) 00000080:00010000:2.0:1769074943.250941:0:41217:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d80733 from it ffffaa9141cfbcd8 00010000:00010000:2.0:1769074943.250944:0:41217:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074943.250953:0:41217:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074943.251327:0:41217:0:(file.c:424:ll_file_release()) START file f12.replay-single:[0x200001b71:0x119:0x0](ffff8b100486e3c8), flags 100000 00000002:00100000:2.0:1769074943.251393:0:41217:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b10037de300 x1855007612453760/t98784247814(98784247814) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074943.251407:0:41217:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074943.251413:0:41217:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59f80 x1855007612456704/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074943.251440:0:41217:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456704:192.168.204.102@tcp:35:multiop.0 00000100:00100000:2.0:1769074943.251536:0:41217:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cd540 going to sleep for 11 seconds 00000100:00100000:2.0:1769074943.263456:0:41217:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job multiop:93598d0a-9484-4e40-94b2-393ee542cbad:41217:1855007612456704:192.168.204.102@tcp:35:multiop.0 00000002:00100000:2.0:1769074943.263503:0:41217:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff8b10037de300 x1855007612453760/t98784247814(98784247814) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 664/608 e 0 to 0 dl 1769074954 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074943.263565:0:41217:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x119:0x0] ignore/verify layout 1/0, layout version -1 restore needed 0 00000080:00200000:2.0:1769074943.263579:0:41217:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x119:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074943.263589:0:41217:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x119:0x0](ffff8b100486e3c8) 00010000:00010000:3.0:1769074943.525408:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074944.544160:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074945.569282:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1769074946.208163:0:37:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1769074946.208570:0:37:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1769074946.208608:0:37:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1769074946.591883:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074946.720399:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612456960:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074946.720538:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612457216:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074946.720583:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1769074946.720590:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612456832:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074946.720626:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612457088:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074946.724167:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612457216:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074946.732518:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612457088:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074947.616629:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074948.640745:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074949.666765:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074950.691511:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074951.713953:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074951.840797:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457472:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074951.840925:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457728:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074951.840968:0:2364:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074951.840975:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612457344:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074951.841010:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074951.847796:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457600:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074951.847821:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457728:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074952.737669:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074953.761393:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074954.783971:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074955.811970:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074956.832314:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074956.964419:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074956.964463:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612457856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074956.964521:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769074956.964535:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612457984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074956.964749:0:2366:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074956.964757:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612458112:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074956.970824:0:2367:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769074956.970856:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612458240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074956.985210:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612458240:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074957.000250:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612458112:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074957.856100:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074958.884211:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074959.904384:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074960.928413:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074961.951925:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769074962.079852:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074946/real 1769074946] req@ffff8b10037dd880 x1855007612456832/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074962 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074962.079902:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037dd880 x1855007612456832/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074962 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074962.079912:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1769074962.079917:0:2366:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1769074962.079921:0:2366:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1769074962.079924:0:2366:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1769074962.079928:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037dd880 x1855007612456832/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074962 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074962.079937:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006136300 x1855007612457344/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074967 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074962.079973:0:2366:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d5ad80 x1855007612457856/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074972 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1769074962.079990:0:2366:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1769074962.079994:0:2366:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1769074962.079996:0:2366:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1769074962.079998:0:2366:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:3.0:1769074962.080011:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074946/real 1769074946] req@ffff8b10037dfb80 x1855007612456960/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074962 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074962.080025:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037dfb80 x1855007612456960/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074962 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074962.080032:0:2366:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1769074962.080036:0:2366:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1769074962.080041:0:2366:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1769074962.080067:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612457856:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074962.080075:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:86724556-597d-4d0e-ac75-6d9d64145050:2366:1855007612456832:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074962.080082:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612456960:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769074962.080223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074962.080231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074962.080235:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074962.080242:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1909 00000100:00080000:0.0:1769074962.080262:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769074962.080306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612458368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769074962.080337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074962.080340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769074962.080342:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769074962.080347:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1909 00000100:00080000:0.0:1769074962.080353:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769074962.080366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769074962.080378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612458496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074962.080452:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458624:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074962.080580:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074962.080619:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074962.081165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612458368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769074962.081245:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612458496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769074962.085430:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458752:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074962.085458:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458624:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074962.086539:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:86724556-597d-4d0e-ac75-6d9d64145050:2364:1855007612457344:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074962.975905:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074964.002401:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074965.025582:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074966.048280:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074967.007782:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074951/real 1769074951] req@ffff8b1006137b80 x1855007612457472/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074967 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074967.007817:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006137b80 x1855007612457472/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074967 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074967.007824:0:2364:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 67, was 66): CONNECTING 00000100:00100000:0.0:1769074967.007847:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612457472:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074967.073628:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769074967.199774:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074967.199788:0:37199:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769074967.199791:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074967.200333:0:37199:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074967.200706:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458880:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074967.200843:0:2365:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074967.200852:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612459008:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074967.206982:0:2365:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:0.0:1769074967.207012:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612459008:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074967.207037:0:2365:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:0.0:1769074967.207043:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612458880:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074967.207083:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074967.207101:0:13997:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074967.207108:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074967.207110:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074967.207112:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074967.207121:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074968.095875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074969.120053:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074969.549947:0:41554:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074969.549964:0:41554:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:0.0:1769074969.549989:0:41554:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074969.550066:0:41554:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 41554 initiator: MDT0 00010000:00010000:0.0:1769074969.550076:0:41554:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41554 initiator: MDT0 00010000:00010000:0.0:1769074969.550087:0:41554:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 41554 initiator: MDT0 00000100:00080000:0.0:1769074969.550117:0:41554:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b1006024700 x1855007612459136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769074969.550128:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442b40 going to sleep for 0 seconds 00010000:00010000:0.0:1769074970.143806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074970.591777:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442b40 going to sleep for 0 seconds 00010000:00010000:0.0:1769074971.167819:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074971.615741:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442b40 going to sleep for 0 seconds 00010000:00010000:3.0:1769074972.191872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769074972.319767:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769074972.319771:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074962/real 1769074962] req@ffff8b1006024e00 x1855007612458368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1769074972.319771:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074956/real 1769074956] req@ffff8b10037df480 x1855007612457984/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074972.319777:0:37199:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769074972.319783:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074972.319793:0:37199:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074972.319829:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612458368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074972.319830:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037df480 x1855007612457984/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074972.319861:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 67, was 66): CONNECTING 00000100:00100000:2.0:1769074972.319865:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074962/real 1769074962] req@ffff8b1006027480 x1855007612458496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074972.319877:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612459264:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074972.319878:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612459392:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074972.319879:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612457984:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074972.319887:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612458496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769074972 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074972.319903:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074972.319921:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074972.319925:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612458368:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769074972.319932:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769074972.319940:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769074972.319944:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612458496:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769074972.319978:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074972.319982:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769074972.319985:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074972.319990:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1955 00000100:00080000:2.0:1769074972.319997:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769074972.320003:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769074972.320022:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612459520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074972.320045:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074972.320049:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769074972.320051:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074972.320056:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1955 00000100:00080000:2.0:1769074972.320061:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769074972.320065:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074972.320073:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769074972.320083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612459520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769074972.320089:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612459648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074972.320128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769074972.321790:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612459392:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074972.321826:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612459264:192.168.204.102@tcp:400:kworker.0 00000100:00080000:3.0:1769074972.321874:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.102@tcp: connect to target with instance 0 10000000:01000000:3.0:1769074972.321899:0:2363:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1769074972.321908:0:2363:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.102@tcp) after server handle changed from 0x143201ebf3064b3d to 0x143201ebf3065022 00000100:00080000:3.0:1769074972.321914:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1769074972.321921:0:2363:0:(import.c:1652:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.102@tcp; invalidating 00000100:00100000:2.0:1769074972.322114:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612459520:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769074972.322145:0:41569:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.102@tcp to MGS@192.168.204.102@tcp 00000100:00100000:3.0:1769074972.322150:0:41569:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1769074972.322152:0:41569:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1769074972.322169:0:41569:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074972.322178:0:41569:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074972.322183:0:41569:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074972.322198:0:41569:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074972.322208:0:41569:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1005698e00/0x68a7113811d806df lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064d2e expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074972.322228:0:41569:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074972.322235:0:41569:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074972.322241:0:41569:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074972.322252:0:41569:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074972.322256:0:41569:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029ad200/0x68a7113811d806ed lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064d3c expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074972.322269:0:41569:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074972.322273:0:41569:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074972.322277:0:41569:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1769074972.322282:0:41569:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074972.322284:0:41569:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b1006169200/0x68a7113811d806d8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064d27 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074972.322298:0:41569:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:00010000:3.0:1769074972.322302:0:41569:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074972.322304:0:41569:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1769074972.322309:0:41569:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074972.322311:0:41569:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.102@tcp lock: ffff8b10029ac000/0x68a7113811d806e6 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0x143201ebf3064d35 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00080000:3.0:1769074972.322326:0:41569:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1769074972.322331:0:41569:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1769074972.322332:0:41569:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1769074972.322337:0:41569:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1769074972.322341:0:41569:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) MGC192.168.204.102@tcp: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00010000:00080000:1.0:1769074972.326883:0:2363:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff8b1005d5b100 x1855007612459648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/416 e 0 to 0 dl 1769074987 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074972.326900:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247817 gen 1 00000100:00100000:1.0:1769074972.326941:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074959 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.326958:0:2363:0:(import.c:1123:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff8b101127b800: connect to target with instance 24 00000100:00080000:1.0:1769074972.326980:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1769074972.326989:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074972.326992:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247817 gen 1 00000100:00100000:1.0:1769074972.327006:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074959 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.327014:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 98784247817 last 0 00000100:00080000:1.0:1769074972.327023:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074932 ref 1 fl New:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769074972.327037:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074972.327068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443392:192.168.204.102@tcp:101:grep.0 00000100:00080000:1.0:1769074972.337352:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000002:00080000:1.0:1769074972.337374:0:2363:0:(mdc_request.c:695:mdc_replay_open()) updating handle from 0x143201ebf3064b67 to 0x143201ebf3065030 00000100:00020000:1.0:1769074972.337378:0:2363:0:(client.c:3418:ptlrpc_replay_interpret()) @@@ status 301, old was 0 req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'grep.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1769074972.347535:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074972.347546:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247817 gen 1 00000100:00100000:1.0:1769074972.347566:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074959 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.347579:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 98784247817 last 94489280524 00000100:00080000:1.0:1769074972.347588:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074959 ref 1 fl New:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074972.347604:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612443392:192.168.204.102@tcp:101:grep.0 00000100:00100000:1.0:1769074972.347650:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612456064:192.168.204.102@tcp:36:multiop.0 00000100:00080000:1.0:1769074972.350319:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.350336:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074972.350339:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247817 gen 1 00000100:00100000:1.0:1769074972.350344:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.350351:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 98784247817 last 98784247818 00000100:00080000:1.0:1769074972.350359:0:2363:0:(client.c:3523:ptlrpc_replay_req()) @@@ REPLAY req@ffff8b1005d59f80 x1855007612456704/t98784247819(98784247819) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074959 ref 1 fl New:RQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769074972.350370:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612456064:192.168.204.102@tcp:36:multiop.0 00000100:00100000:1.0:1769074972.350402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612456704:192.168.204.102@tcp:35:multiop.0 00000100:00080000:1.0:1769074972.351898:0:2363:0:(client.c:3408:ptlrpc_replay_interpret()) @@@ got reply req@ffff8b1005d59f80 x1855007612456704/t98784247819(98784247819) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RQU/604/0 rc -116/-116 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00020000:1.0:1769074972.351913:0:2363:0:(client.c:3418:ptlrpc_replay_interpret()) @@@ status -116, old was 0 req@ffff8b1005d59f80 x1855007612456704/t98784247819(98784247819) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074988 ref 2 fl Interpret:RQU/604/0 rc -116/-116 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1769074972.351925:0:2363:0:(import.c:1679:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1769074972.351928:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247817 gen 1 00000100:00100000:1.0:1769074972.351933:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1006137800 x1855007612456064/t98784247818(98784247818) o36->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 488/456 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.351940:0:2363:0:(recover.c:53:ptlrpc_replay_next()) import ffff8b1008679800 from lustre-MDT0000_UUID committed 98784247817 last 98784247819 00000100:00080000:1.0:1769074972.351945:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1769074972.352170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612456704:192.168.204.102@tcp:35:multiop.0 00010000:00080000:2.0:1769074972.352196:0:41570:0:(ldlm_request.c:2714:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffff8b101127b800 to lustre-MDT0000_UUID@192.168.204.102@tcp 00010000:00010000:2.0:1769074972.352201:0:41570:0:(ldlm_request.c:2634:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffff8b101127b800 (5) 00010000:00010000:2.0:1769074972.352233:0:41570:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698000/0x68a7113811d80733 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x119:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3064e77 expref: -99 pid: 41217 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.352250:0:41570:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1006169800/0x68a7113811d8072c lrc: 0/0,0 mode: --/CR res: [0x200001b71:0x119:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3064e69 expref: -99 pid: 41217 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074972.352283:0:41570:0:(vvp_object.c:114:vvp_conf_set()) [0x200001b71:0x115:0x0]: losing layout lock 00010000:00010000:2.0:1769074972.352312:0:41570:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005698600/0x68a7113811d80702 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3064d9e expref: -99 pid: 40641 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074972.352347:0:41570:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064b6e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074972.352370:0:41570:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac400/0x68a7113811d806fb lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0x143201ebf3064d7b expref: -99 pid: 40636 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074972.352379:0:41570:0:(ldlm_request.c:2647:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffff8b101127b800 00010000:00010000:2.0:1769074972.352863:0:41570:0:(ldlm_request.c:2606:replay_one_lock()) ### replaying lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf3064b6e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00010000:00080000:2.0:1769074972.352888:0:41570:0:(ldlm_request.c:2656:lock_can_replay()) check lock replay limit, inflights = 1(8) 00000100:00100000:1.0:1769074972.352923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459776:192.168.204.102@tcp:101:ldlm_lock_repla.0 00010000:00010000:1.0:1769074972.354204:0:2363:0:(ldlm_request.c:2523:replay_lock_interpret()) ### replayed lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x143201ebf306503e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000100:00080000:1.0:1769074972.354218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:1.0:1769074972.354245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459776:192.168.204.102@tcp:101:ldlm_lock_repla.0 00000100:00100000:1.0:1769074972.354271:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459904:192.168.204.102@tcp:400:ptlrpcd_rcv.0 00000100:00100000:1.0:1769074972.363946:0:2363:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff8b101127b800: committing for last_committed 98784247818 gen 1 00000100:00100000:1.0:1769074972.363986:0:2363:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff8b1005d59f80 x1855007612456704/t98784247819(98784247819) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/456 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/604/0 rc 0/-116 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1769074972.364004:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:1.0:1769074972.364010:0:2363:0:(client.c:3203:ptlrpc_resend_req()) @@@ going to resend req@ffff8b1006136a00 x1855007612459904/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074988 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.364018:0:2363:0:(client.c:3212:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffff8b1006136a00 x1855007612459904/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074988 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074972.364025:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1769074972.364378:0:2363:0:(import.c:1722:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffff8b101127b800: Connection restored to 192.168.204.102@tcp (at 192.168.204.102@tcp) 00000100:00080000:1.0:1769074972.364385:0:2363:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 00000000b828ecaf): req@ffff8b1006024700 x1855007612459136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1769074972.364406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612459904:192.168.204.102@tcp:400:ptlrpcd_rcv.0 00000100:00100000:0.0:1769074972.364575:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442b40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074972.366382:0:41554:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612459136:192.168.204.102@tcp:101:lfs.0 00010000:00010000:2.0:1769074972.366410:0:41554:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366434:0:41554:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366452:0:41554:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366466:0:41554:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366478:0:41554:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366484:0:41554:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366490:0:41554:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074972.366497:0:41554:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff8b1006024700 x1855007612459136/t0(0) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074972.366512:0:41554:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366526:0:41554:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff8b100a70d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074972.366534:0:41554:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074972.366566:0:41554:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 18944 00000080:00010000:2.0:1769074972.366574:0:41554:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8073a 00000080:00010000:2.0:1769074972.366581:0:41554:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074972.366585:0:41554:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8073a from it ffffaa9141ed3cb8 00010000:00010000:2.0:1769074972.366588:0:41554:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.366597:0:41554:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074972.367382:0:41554:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074972.367389:0:41554:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074972.367407:0:41554:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074972.367428:0:41554:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.367441:0:41554:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074972.367448:0:41554:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074972.367455:0:41554:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074972.367458:0:41554:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8073a 00000080:00010000:2.0:1769074972.367462:0:41554:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8073a from it ffffaa9141ed3cb8 00010000:00010000:2.0:1769074972.367465:0:41554:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.367471:0:41554:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074972.367574:0:41554:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08),name=/ 00000002:00010000:2.0:1769074972.367582:0:41554:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074972.367589:0:41554:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.367597:0:41554:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074972.367605:0:41554:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074972.367609:0:41554:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) 00000080:00010000:2.0:1769074972.367612:0:41554:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1018194b08) for lock 0x68a7113811d8073a 00000080:00010000:2.0:1769074972.367616:0:41554:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x68a7113811d8073a from it ffffaa9141ed3bb0 00010000:00010000:2.0:1769074972.367619:0:41554:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074972.367626:0:41554:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074972.367634:0:41554:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08), inode mode 41ed mask 44 00000080:00200008:2.0:1769074972.367661:0:41554:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00000080:00200080:2.0:1769074972.367755:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.367773:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00800000:00000004:2.0:1769074972.367781:0:41554:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: age 1932, max_age 1964 00000100:00100000:2.0:1769074972.367825:0:41554:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460032:192.168.204.102@tcp:41:lfs.0 00000100:00100000:2.0:1769074972.367916:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cde40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074972.368950:0:41554:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460032:192.168.204.102@tcp:41:lfs.0 00800000:00000004:2.0:1769074972.368982:0:41554:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff8b101127b800: update ffff8b1007d04df8 cache blocks 17229/17271 objects 501048/501398 00000080:00200080:2.0:1769074972.369032:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369041:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369046:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369050:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369054:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369056:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369060:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369063:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369065:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369067:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369070:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369071:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369073:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369075:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369078:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369081:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369084:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369088:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369092:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369095:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369099:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369102:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369106:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369110:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369114:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369117:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369122:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369126:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369130:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369134:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369140:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369145:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369150:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369155:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369159:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369164:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369169:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369172:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369176:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369179:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369183:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369188:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369192:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369197:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369201:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369205:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369210:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369214:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369220:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369224:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369228:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369232:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369236:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369238:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369248:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369252:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369257:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369261:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369265:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369269:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369274:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369278:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369283:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369286:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369292:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369296:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369299:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369303:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369307:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369310:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369315:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369318:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369322:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369325:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369328:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369332:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369336:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369339:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369344:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369347:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369351:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369355:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369359:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369362:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369366:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369370:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369375:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369377:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369380:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369382:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369384:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369386:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369387:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369389:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369391:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369393:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369396:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369400:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369404:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369407:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369411:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369424:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369429:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369433:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369436:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369440:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369444:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369448:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369453:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369456:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369459:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369463:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369467:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369471:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369475:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369478:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369483:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369486:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369490:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369493:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369498:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369501:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369505:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369509:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369513:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369517:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369521:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00800000:00000080:2.0:1769074972.369524:0:41554:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.369528:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.369534:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00020000:00000004:2.0:1769074972.369543:0:41554:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: age 1932, max_age 1964 00000100:00100000:2.0:1769074972.369578:0:41554:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460160:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074972.369667:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cde40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074972.370571:0:41554:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460160:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074972.370591:0:41554:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff8b101127b800: update ffff8b100a728370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:2.0:1769074972.370633:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.370666:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00020000:00000004:2.0:1769074972.370672:0:41554:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: age 1932, max_age 1964 00000100:00100000:2.0:1769074972.370694:0:41554:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460288:192.168.204.102@tcp:13:lfs.0 00000100:00100000:2.0:1769074972.370742:0:41554:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10088cde40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074972.371457:0:41554:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job lfs:93598d0a-9484-4e40-94b2-393ee542cbad:41554:1855007612460288:192.168.204.102@tcp:13:lfs.0 00020000:00000004:2.0:1769074972.371466:0:41554:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff8b101127b800: update ffff8b1007d00370 cache blocks 3678/3683 objects 117760/118191 00000080:00200080:2.0:1769074972.371494:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371503:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371509:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371513:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371519:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371523:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371528:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371532:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371537:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371542:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371547:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371558:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371564:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371569:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371574:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371578:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371583:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371587:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371591:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371595:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371600:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371604:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371610:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371614:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371619:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371621:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371624:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371627:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371630:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371631:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371634:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371636:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371658:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371660:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371662:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371665:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371668:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371671:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371674:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371678:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371680:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371682:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371685:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371686:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371689:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371691:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371694:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371695:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371698:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371700:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371702:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371704:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371707:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371709:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371711:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371713:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371715:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371717:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371720:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371721:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371724:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371727:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371730:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371733:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371734:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371736:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371738:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371740:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371743:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371745:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371748:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371750:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371752:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371753:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371755:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371757:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371759:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371760:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371762:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371764:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371766:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371767:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371769:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371771:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371773:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371774:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371776:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371778:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371780:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371782:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371784:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371786:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371787:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371789:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371791:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371793:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371794:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371805:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371809:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371812:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371815:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371818:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371821:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371824:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371827:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371830:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371833:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371837:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371841:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371844:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371849:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371852:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371857:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371860:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371862:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371865:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371867:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371869:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371871:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371872:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371874:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371876:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371877:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371879:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200080:2.0:1769074972.371881:0:41554:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1018194b08) cmd=c00866a4 arg=7fff8132b400 00020000:00000080:2.0:1769074972.371883:0:41554:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff8b101127b800: cmd=c00866a4 len=592 karg=ffff8b10096aac00 uarg=0000000000000000 00000080:00200008:2.0:1769074972.371895:0:41554:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff8b1018194b08), flags 100000 00010000:00010000:2.0:1769074973.215745:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1769074973.407729:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:0.0:1769074973.407736:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff8b101127b800 from 29 10000000:01000000:0.0:1769074973.407740:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1769074973.407778:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074973.407787:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074973.407794:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074973.407815:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460416:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1769074973.407886:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1009442540 going to sleep for 11 seconds 00000100:00100000:3.0:1769074973.409100:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460416:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074973.409126:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065213 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074973.409137:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3065213 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074973.409214:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460544:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1769074973.409291:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1007cb0cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074973.410701:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460544:192.168.204.102@tcp:256:ll_cfg_requeue.0 00000020:00000080:3.0:1769074973.410751:0:14000:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00f 10000000:01000000:3.0:1769074973.410885:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1769074973.410893:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065213 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074973.410902:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10029ac600/0x68a7113811d80741 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065213 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:3.0:1769074973.410911:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1769074973.410913:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-client-ffff8b101127b800 from 42 10000000:01000000:3.0:1769074973.410916:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1769074973.410946:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074973.410955:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074973.410964:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074973.410981:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460672:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1769074973.411035:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1007cb0cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074973.411912:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460672:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1769074973.411929:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306521a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074973.411938:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf306521a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:3.0:1769074973.411995:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460800:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074973.412053:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1007cb0cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074973.412830:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460800:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1769074973.412861:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460928:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1769074973.412911:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1007cb0cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1769074973.413833:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612460928:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1769074973.414570:0:41778:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 42 00000020:01000000:1.0:1769074973.414660:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 42-41 (rc=0) 10000000:01000000:1.0:1769074973.414692:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1769074973.414704:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306521a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074973.414722:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1006168a00/0x68a7113811d80748 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf306521a expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:1.0:1769074973.414733:0:14000:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:1.0:1769074973.414735:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log params-ffff8b101127b800 from 7 10000000:01000000:1.0:1769074973.414738:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:1.0:1769074973.414768:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074973.414775:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074973.414782:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074973.414805:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461056:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1769074973.414866:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b1011c56240 going to sleep for 11 seconds 00000100:00100000:2.0:1769074973.415757:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461056:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074973.415772:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065221 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.415781:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3065221 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074973.415827:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461184:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074973.415874:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a03c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074973.416737:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461184:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074973.416798:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461312:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1769074973.416879:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a03c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074973.417729:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461312:192.168.204.102@tcp:503:ll_cfg_requeue.0 00000040:00080000:0.0:1769074973.418392:0:41780:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:2.0:1769074973.418472:0:14000:0:(obd_config.c:2043:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:2.0:1769074973.418510:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1769074973.418520:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065221 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.418528:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b10060b9600/0x68a7113811d8074f lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065221 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 10000000:01000000:2.0:1769074973.418536:0:14000:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1769074973.418538:0:14000:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-sptlrpc-ffff8b10112fe360 from 1 10000000:01000000:2.0:1769074973.418539:0:14000:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1769074973.418563:0:14000:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.418569:0:14000:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.418576:0:14000:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074973.418588:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461440:192.168.204.102@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1769074973.418637:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a03c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074973.419554:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461440:192.168.204.102@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1769074973.419569:0:14000:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065228 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.419578:0:14000:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x143201ebf3065228 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074973.419618:0:14000:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461568:192.168.204.102@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1769074973.419695:0:14000:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b10036a03c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074973.420456:0:14000:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:86724556-597d-4d0e-ac75-6d9d64145050:14000:1855007612461568:192.168.204.102@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1769074973.420501:0:14000:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.102@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1769074973.420511:0:14000:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065228 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074973.420520:0:14000:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.102@tcp lock: ffff8b1005698800/0x68a7113811d80756 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x143201ebf3065228 expref: -99 pid: 14000 timeout: 0 lvb_type: 0 00000001:02000400:2.0:1769074973.605422:0:41810:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg402-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000001:02000400:3.0:1769074974.168777:0:41964:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:0.0:1769074974.239738:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074975.263841:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074976.287797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074977.311881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074977.439859:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612461824:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074977.439859:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612461696:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074977.439962:0:2365:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612461952:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074977.439963:0:2364:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612462080:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074978.335811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074979.359858:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074980.383801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074981.407866:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074982.431856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074982.559930:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612462208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074982.559930:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462336:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074982.560045:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612462464:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074982.560045:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462592:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074983.455899:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074984.479955:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074985.503886:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074986.527813:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074987.551833:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074987.679861:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074987.679866:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612462720:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074987.679963:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074987.679966:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612462976:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769074988.575836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074989.599811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074990.623864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074991.647820:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074992.671848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074992.799866:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612463232:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074992.799986:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463360:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074992.799991:0:2367:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612463488:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769074992.800141:0:2366:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463616:192.168.204.102@tcp:400:kworker.0 00010000:00010000:2.0:1769074993.695775:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074993.823777:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff8b1006025f80 x1855007612461824/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1769074993.823777:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff8b1006026d80 x1855007612461696/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074993.823835:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006026d80 x1855007612461696/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074993.823838:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006025f80 x1855007612461824/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074993.823859:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from FULL to DISCONN 00000100:00080000:0.0:1769074993.823860:0:2364:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1769074993.823869:0:2364:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff8b101127b800: Connection to lustre-MDT0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:02020000:1.0:1769074993.823870:0:2365:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.102@tcp: Connection to MGS (at 192.168.204.102@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1769074993.823875:0:2365:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.102@tcp for MGC192.168.204.102@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1769074993.823877:0:2365:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00080000:0.0:1769074993.823885:0:2364:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1769074993.823885:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1006026d80 x1855007612461696/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074993.823897:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037df480 x1855007612462208/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074998 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1769074993.823907:0:2364:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff8b1006026300 x1855007612462080/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074993.823915:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b10037dce00 x1855007612462720/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769075003 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074993.823929:0:2364:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006026300 x1855007612462080/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074993.823929:0:2365:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff8b1005d59880 x1855007612463232/t0(0) o400->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 224/224 e 0 to 0 dl 1769075008 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074993.823951:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612462720:192.168.204.102@tcp:400:kworker.0 10000000:01000000:1.0:1769074993.823962:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1769074993.823963:0:2364:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:00080000:2.0:1769074993.823963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074993.823966:0:2365:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1769074993.823969:0:2365:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:02000400:0.0:1769074993.823974:0:2364:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff8b101127b800: Connection to lustre-OST0001 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074993.823975:0:2365:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:2.0:1769074993.823976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769074993.823982:0:2364:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00080000:2.0:1769074993.823984:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074993.823997:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1965 00000100:00000400:1.0:1769074993.823998:0:2365:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff8b1006024a80 x1855007612461952/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074993.823999:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612462208:192.168.204.102@tcp:400:kworker.0 00000100:00100000:0.0:1769074993.824004:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612461824:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769074993.824023:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769074993.824026:0:2365:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1006024a80 x1855007612461952/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074993.824040:0:2365:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074993.824044:0:2365:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff8b101127b800: Connection to lustre-OST0000 (at 192.168.204.102@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769074993.824048:0:2365:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1769074993.824070:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:86724556-597d-4d0e-ac75-6d9d64145050:2367:1855007612463232:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769074993.824077:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:86724556-597d-4d0e-ac75-6d9d64145050:2365:1855007612461696:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769074993.824087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612463744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.824108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.824111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769074993.824113:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074993.824117:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1965 00000100:00080000:2.0:1769074993.824123:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769074993.824128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612463744:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769074993.824134:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769074993.824148:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612463872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.824157:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.824159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769074993.824161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074993.824164:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1857 00000100:00080000:2.0:1769074993.824169:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074993.824172:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769074993.824180:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612464000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.824187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.824189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769074993.824190:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074993.824193:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1857 00000100:00080000:2.0:1769074993.824196:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769074993.824199:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769074993.824207:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612464128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.824217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.824221:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769074993.824232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612463872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769074993.824261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769074993.824293:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464128:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769074994.719819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074995.743769:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074996.767874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074997.791828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074998.815832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769074998.943744:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff8b10037dfb80 x1855007612462336/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1769074998.943744:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff8b10037dd880 x1855007612462464/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074998.943746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074998.943756:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769074998.943762:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074998.943770:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769074998.943778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074998.943787:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769074998.943789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769074998.943791:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074998.943813:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037dfb80 x1855007612462336/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074998.943817:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037dd880 x1855007612462464/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074998.943835:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 69, was 68): CONNECTING 00000100:00080000:3.0:1769074998.943837:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1011207000 already not connected (conn 11, was 10): CONNECTING 00000100:00000400:2.0:1769074998.943846:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff8b10037df800 x1855007612462592/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074998.943859:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037df800 x1855007612462592/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074998.943864:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1008678000 already not connected (conn 11, was 10): CONNECTING 00000100:00100000:2.0:1769074998.943875:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462336:192.168.204.102@tcp:400:kworker.0 00010000:00010000:0.0:1769074999.839875:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075000.863859:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075001.887902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075002.911892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075003.039752:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff8b1005d5bb80 x1855007612463872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1769075003.039752:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074987/real 1769074987] req@ffff8b10037dea00 x1855007612462976/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1769075003.039799:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074987/real 1769074987] req@ffff8b10037ddc00 x1855007612462848/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075003.039810:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037dea00 x1855007612462976/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075003.039820:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1011207000 already not connected (conn 11, was 10): CONNECTING 00000100:00100000:1.0:1769075003.039821:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612463872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075003.039824:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037ddc00 x1855007612462848/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075003.039854:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 69, was 68): CONNECTING 00000100:00100000:1.0:1769075003.039861:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff8b1005d58e00 x1855007612464000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1769075003.039863:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074987/real 1769074987] req@ffff8b10037df100 x1855007612463104/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075003.039896:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b10037df100 x1855007612463104/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075003.039918:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612464000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075003.039937:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1008678000 already not connected (conn 11, was 10): CONNECTING 00000100:00100000:1.0:1769075003.039941:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff8b1005d59500 x1855007612464128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075003.039950:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462848:192.168.204.102@tcp:400:kworker.0 00000100:00100000:1.0:1769075003.039954:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612464128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075003.039964:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff8b1005d5b800 x1855007612463744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075003.039977:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612463744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075003 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075003.039989:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075003.040009:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075003.040013:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612463872:192.168.204.102@tcp:38:kworker.0 00000100:00080000:1.0:1769075003.040031:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075003.040038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.040043:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769075003.040046:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00080000:2.0:1769075003.040048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075003.040053:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464000:192.168.204.102@tcp:8:kworker.0 00000100:00080000:2.0:1769075003.040054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075003.040059:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075003.040066:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1987 00000100:00100000:3.0:1769075003.040073:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612462976:192.168.204.102@tcp:400:kworker.0 00000100:00080000:1.0:1769075003.040074:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769075003.040076:0:2365:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:93598d0a-9484-4e40-94b2-393ee542cbad:2365:1855007612461952:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769075003.040079:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00100000:3.0:1769075003.040088:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612462464:192.168.204.102@tcp:400:kworker.0 00000100:00080000:1.0:1769075003.040089:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00080000:2.0:1769075003.040091:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075003.040101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464128:192.168.204.102@tcp:8:kworker.0 00000008:00100000:2.0:1769075003.040108:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1769075003.040109:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075003.040118:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075003.040121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612463744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075003.040143:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463104:192.168.204.102@tcp:400:kworker.0 00000100:00100000:2.0:1769075003.040146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612464256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075003.040153:0:2364:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:93598d0a-9484-4e40-94b2-393ee542cbad:2364:1855007612462080:192.168.204.102@tcp:400:kworker.0 00000100:00100000:3.0:1769075003.040157:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612462592:192.168.204.102@tcp:400:kworker.0 00000100:00080000:2.0:1769075003.040181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.040184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075003.040185:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075003.040189:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1987 00000100:00080000:2.0:1769075003.040193:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769075003.040197:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075003.040202:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075003.040210:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612464384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075003.040215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464256:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769075003.040223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.040227:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075003.040228:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075003.040231:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1987 00000100:00080000:2.0:1769075003.040233:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769075003.040235:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075003.040238:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075003.040247:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612464512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075003.040256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.040258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075003.040259:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075003.040262:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1987 00000100:00080000:2.0:1769075003.040266:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769075003.040269:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075003.040278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a680 x1855007612464640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075003.040327:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075003.040351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075003.040365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612464640:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769075003.935854:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075004.959801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075005.983864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075007.007805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075008.031863:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769075008.159707:0:2367:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074992/real 1769074992] req@ffff8b1005d59180 x1855007612463488/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.159748:0:2367:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d59180 x1855007612463488/t0(0) o400->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1769075008.159758:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074992/real 1769074992] req@ffff8b1005d59c00 x1855007612463360/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075008.159758:0:2367:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1011207000 already not connected (conn 12, was 10): CONNECTING 00000100:00080000:0.0:1769075008.159774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075008.159779:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075008.159782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075008.159787:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d59c00 x1855007612463360/t0(0) o400->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075008.159794:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075008.159803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075008.159804:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff8b1008679800 already not connected (conn 70, was 68): CONNECTING 00000100:00080000:0.0:1769075008.159824:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075008.159829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000400:2.0:1769075008.159830:0:2366:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074992/real 1769074992] req@ffff8b1005d58a80 x1855007612463616/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075008.159835:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769075008.159856:0:2366:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1005d58a80 x1855007612463616/t0(0) o400->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 224/224 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075008.159874:0:2366:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff8b1008678000 already not connected (conn 12, was 10): CONNECTING 00000100:00100000:2.0:1769075008.159889:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463360:192.168.204.102@tcp:400:kworker.0 00010000:00010000:3.0:1769075009.055859:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075010.079899:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075011.103831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1769075011.615820:0:2355:0:(socklnd_cb.c:2367:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.204.102@tcp (192.168.204.102:988) timed out; the network or node may be down. 00010000:00010000:0.0:1769075012.127883:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075013.151823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075013.279823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075013.279831:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769075013.279834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075013.279837:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769075013.279838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075013.279841:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769075013.279844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075013.279847:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1769075014.175789:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075015.199750:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075016.223765:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075017.247767:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075018.271837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075018.399743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075018.399744:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff8b1005d5b480 x1855007612464384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075018.399756:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075018.399763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.399768:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075018.399770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.399772:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075018.399774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.399776:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769075018.399791:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612464384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399801:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff8b1005d5a300 x1855007612464512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399815:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612464512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399822:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff8b1005d5a680 x1855007612464640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399829:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a680 x1855007612464640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399835:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff8b1005d58000 x1855007612464256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075018.399843:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612464256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075018 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075018.399869:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075018.399881:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075018.399885:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464384:192.168.204.102@tcp:8:kworker.0 00000100:00080000:1.0:1769075018.399895:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075018.399901:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00080000:0.0:1769075018.399905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075018.399906:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464512:192.168.204.102@tcp:8:kworker.0 00000100:00080000:0.0:1769075018.399911:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075018.399915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075018.399917:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075018.399918:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769075018.399922:0:2367:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:93598d0a-9484-4e40-94b2-393ee542cbad:2367:1855007612463488:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769075018.399925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075018.399929:0:2366:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:93598d0a-9484-4e40-94b2-393ee542cbad:2366:1855007612463616:192.168.204.102@tcp:400:kworker.0 00000100:00080000:0.0:1769075018.399936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075018.399938:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00080000:0.0:1769075018.399942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075018.399944:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612464640:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075018.399957:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1996 00000100:00080000:1.0:1769075018.399957:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075018.399972:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00080000:0.0:1769075018.399978:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 10s 00000100:00100000:1.0:1769075018.399978:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464256:192.168.204.102@tcp:38:kworker.0 00000100:00080000:0.0:1769075018.399992:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075018.400009:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075018.400051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612464768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075018.400071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.400073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075018.400075:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075018.400078:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1996 00000100:00080000:0.0:1769075018.400081:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075018.400085:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075018.400089:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075018.400097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612464896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075018.400108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.400113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1769075018.400113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464768:192.168.204.102@tcp:8:kworker.0 00000100:00080000:0.0:1769075018.400116:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075018.400121:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 1996 00000100:00080000:0.0:1769075018.400123:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075018.400125:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075018.400136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612465024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075018.400143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075018.400147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075018.400149:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075018.400151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 1996 00000100:00080000:0.0:1769075018.400153:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075018.400156:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075018.400160:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075018.400168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612465152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075018.400243:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075018.400279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075018.400312:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465152:192.168.204.102@tcp:38:kworker.0 00010000:00010000:0.0:1769075019.295804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075020.319810:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075021.088095:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:3.0:1769075021.088156:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075021.088180:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075021.088233:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075021.088244:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075021.088254:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075021.088391:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075018/real 1769075021] req@ffff8b1006024a80 x1855007612464896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075021.088424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612464896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075021.088440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075021.088447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075021.088452:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075021.088462:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075018/real 1769075021] req@ffff8b1006025f80 x1855007612465024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075021.088477:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612465024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075021.088486:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075021.088490:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075021.088493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075021.088504:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075018/real 1769075021] req@ffff8b1006026a00 x1855007612465152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075021.088518:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612465152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075021.088525:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075021.088530:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075021.088533:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075021.088540:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075018/real 1769075021] req@ffff8b1006026300 x1855007612464768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075021.088553:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612464768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075021.088561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075021.088564:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075021.088568:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612464768:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075021.343745:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075022.367817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075023.391858:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075023.519798:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075023.519809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075023.519814:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075023.519823:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2011 00000100:00080000:2.0:1769075023.519835:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1769075023.519844:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075023.519880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612465280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075023.519899:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075023.519902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075023.519903:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075023.519906:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2011 00000100:00080000:2.0:1769075023.519908:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1769075023.519914:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075023.519915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465280:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769075023.519921:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075023.519933:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59c00 x1855007612465408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075023.519941:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075023.519945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075023.519948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075023.519952:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2011 00000100:00080000:2.0:1769075023.519957:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1769075023.519961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075023.519966:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075023.519979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612465536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075023.519991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075023.520002:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465408:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769075023.520006:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075023.520010:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075023.520015:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2011 00000100:00080000:2.0:1769075023.520022:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1769075023.520025:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075023.520031:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465536:192.168.204.102@tcp:8:kworker.0 00000008:00100000:2.0:1769075023.520031:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075023.520049:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a680 x1855007612465664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075023.520072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465664:192.168.204.102@tcp:8:kworker.0 00000400:00000100:0.0:1769075024.159887:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:0.0:1769075024.159928:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769075024.159943:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769075024.159972:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769075024.159979:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769075024.159984:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075024.160032:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075023/real 1769075024] req@ffff8b1005d5a680 x1855007612465664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075024.160085:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a680 x1855007612465664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075024.160097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075024.160104:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075024.160109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075024.160133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075023/real 1769075024] req@ffff8b1005d59c00 x1855007612465408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075024.160146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59c00 x1855007612465408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075024.160154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075024.160158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075024.160162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075024.160168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075023/real 1769075024] req@ffff8b1005d58000 x1855007612465536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075024.160180:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612465536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075024.160188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075024.160191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075024.160194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075024.160201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075023/real 1769075024] req@ffff8b1005d58a80 x1855007612465280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075024.160213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612465280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075024.160220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075024.160224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075024.160227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465280:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769075024.415865:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075025.439791:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075026.463758:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075027.487793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075028.511833:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075028.639738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075028.639747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075028.639751:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075028.639759:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2016 00000100:00080000:2.0:1769075028.639769:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075028.639775:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075028.639805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612465792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075028.639837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075028.639841:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075028.639843:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075028.639849:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2016 00000100:00080000:2.0:1769075028.639855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075028.639858:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075028.639866:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075028.639879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612465920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075028.639887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075028.639890:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075028.639892:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075028.639896:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2016 00000100:00080000:2.0:1769075028.639898:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075028.639901:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075028.639906:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075028.639911:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075028.639916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612466048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075028.639931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075028.639934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075028.639936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075028.639939:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2016 00000100:00080000:2.0:1769075028.639943:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075028.639948:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075028.639954:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075028.639969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612466176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075028.640039:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075028.640141:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075028.640178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466176:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075029.535792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075030.304005:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:2.0:1769075030.304066:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075030.304091:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075030.304164:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075030.304178:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075030.304188:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075030.304411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff8b1005d5b480 x1855007612466048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075030.304458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612466048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075030.304472:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075030.304478:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075030.304483:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075030.304492:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff8b1005d5b800 x1855007612466176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075030.304506:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612466176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075030.304526:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075030.304531:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075030.304537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075030.304545:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff8b1005d5a300 x1855007612465792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075030.304561:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612465792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075030.304569:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075030.304573:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075030.304577:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612465792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075030.304584:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff8b1005d59180 x1855007612465920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075030.304597:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612465920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075030.304604:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075030.304608:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075030.304611:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612465920:192.168.204.102@tcp:38:kworker.0 00010000:00010000:0.0:1769075030.559780:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075031.583772:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075032.607752:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075033.631782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075033.759735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075033.759744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075033.759748:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075033.759757:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2021 00000100:00080000:0.0:1769075033.759766:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075033.759771:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075033.759803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612466304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075033.759827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075033.759830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075033.759831:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075033.759834:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2021 00000100:00080000:0.0:1769075033.759836:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075033.759839:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075033.759846:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075033.759856:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612466432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075033.759862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075033.759865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075033.759873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075033.759876:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2021 00000100:00080000:0.0:1769075033.759880:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075033.759883:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075033.759887:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075033.759895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612466560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075033.759909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075033.759913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769075033.759914:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612466304:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075033.759915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075033.759925:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2021 00000100:00080000:0.0:1769075033.759931:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075033.759942:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075033.759948:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075033.759964:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612466688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075033.760017:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075033.760061:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075033] req@ffff8b1006024000 x1855007612466304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075033.760085:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612466304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075033.760106:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075033.760112:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075033.760116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612466304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075033.760123:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466432:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075033.760164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075033.760175:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075033] req@ffff8b1006025880 x1855007612466432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075033.760186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612466432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075033.760192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075033.760195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075033.760197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075033.760239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075033.760253:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075033.760260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075033] req@ffff8b1006027800 x1855007612466560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075033.760268:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612466560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075033.760274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075033.760276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075033.760278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075033.760283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075033.760294:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075033.760300:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075033] req@ffff8b1006027100 x1855007612466688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075033.760308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612466688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075033.760314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075033.760316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075033.760318:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466688:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075034.655777:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075035.679853:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075036.703736:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075037.727816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075038.751774:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075038.879757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075038.879768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075038.879773:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075038.879783:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2026 00000100:00080000:0.0:1769075038.879794:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075038.879801:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075038.879839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612466816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075038.879871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075038.879874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075038.879876:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075038.879879:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2026 00000100:00080000:0.0:1769075038.879882:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075038.879885:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075038.879894:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075038.879904:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612466944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075038.879912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075038.879916:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075038.879918:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075038.879921:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2026 00000100:00080000:0.0:1769075038.879923:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075038.879925:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075038.879928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075038.879939:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612467072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075038.879946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075038.879951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075038.879956:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075038.879957:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612466816:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075038.879962:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2026 00000100:00080000:0.0:1769075038.879975:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075038.879979:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075038.879983:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075038.879991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612467200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075038.880108:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075038.880165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075038.880193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467200:192.168.204.102@tcp:8:kworker.0 00000400:00000100:2.0:1769075039.520072:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:2.0:1769075039.520139:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075039.520163:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075039.520225:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075039.520236:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075039.520245:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075039.520361:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075039] req@ffff8b1006027100 x1855007612467072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075039.520394:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612467072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075039.520408:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075039.520415:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075039.520422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075039.520431:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075039] req@ffff8b1006027800 x1855007612467200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075039.520446:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612467200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075039.520455:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075039.520459:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075039.520463:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075039.520470:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075039] req@ffff8b1006025180 x1855007612466816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075039.520484:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612466816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075039.520492:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075039.520497:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075039.520500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612466816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075039.520508:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075039] req@ffff8b1006026680 x1855007612466944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075039.520521:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612466944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075039.520530:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075039.520534:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075039.520539:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612466944:192.168.204.102@tcp:38:kworker.0 00010000:00010000:0.0:1769075039.775789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075040.799859:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075041.823799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075042.847769:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075043.871809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075043.999787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075043.999807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075043.999812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075043.999824:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2032 00000100:00080000:1.0:1769075043.999838:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075043.999846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075043.999907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612467328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075043.999942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075043.999946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075043.999948:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075043.999954:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2032 00000100:00080000:1.0:1769075043.999960:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075043.999966:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075043.999978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075043.999986:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612467328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075043.999996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612467456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075044.000018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075044.000022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075044.000025:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075044.000029:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2032 00000100:00080000:1.0:1769075044.000034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075044.000038:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075044.000044:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075044.000061:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075044.000063:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612467584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075044.000077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075044.000082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075044.000085:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075044.000090:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2032 00000100:00080000:1.0:1769075044.000095:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075044.000101:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075044.000104:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075044] req@ffff8b1006137800 x1855007612467328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769075044.000110:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075044.000122:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612467328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075044.000129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612467712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075044.000134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075044.000143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075044.000150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612467328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075044.000175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075044.000192:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075044.000201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075044] req@ffff8b1006135180 x1855007612467456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075044.000210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612467456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075044.000229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075044.000231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075044.000234:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075044.000238:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075044.000255:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075044.000262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075044] req@ffff8b1006136a00 x1855007612467584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075044.000271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612467584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075044.000276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075044.000278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075044.000280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075044.000284:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075044.000294:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075044.000300:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075044] req@ffff8b1006136300 x1855007612467712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075044.000308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612467712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075044.000313:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075044.000315:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075044.000317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467712:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075044.895781:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075045.919760:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075046.943814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075047.967800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075048.991825:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075049.119734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075049.119746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075049.119752:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075049.119766:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2037 00000100:00080000:0.0:1769075049.119780:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075049.119789:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075049.119840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612467840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075049.119867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075049.119870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075049.119872:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075049.119876:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2037 00000100:00080000:0.0:1769075049.119879:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075049.119883:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075049.119892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075049.119905:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612467968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075049.119915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612467840:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075049.119917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075049.119925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075049.119927:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075049.119930:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2037 00000100:00080000:0.0:1769075049.119932:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075049.119935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075049.119938:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075049.119946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612468096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075049.119954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075049.119957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075049.119958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075049.119961:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2037 00000100:00080000:0.0:1769075049.119963:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075049.119965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075049.119968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075049.119974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612468224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075049.120036:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075049.120066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075049.120081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468224:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075050.015807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075051.039810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075051.808075:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:1.0:1769075051.808128:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075051.808143:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075051.808176:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075051.808186:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075051.808195:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075051.808242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075051] req@ffff8b1006024000 x1855007612467968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075051.808269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612467968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075051.808282:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075051.808288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075051.808293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612467968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075051.808302:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075051] req@ffff8b1006026300 x1855007612468096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075051.808316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612468096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075051.808325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075051.808329:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075051.808333:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075051.808341:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075051] req@ffff8b1006026a00 x1855007612468224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075051.808355:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612468224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075051.808363:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075051.808367:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075051.808371:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075051.808379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075051] req@ffff8b1006025880 x1855007612467840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075051.808392:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612467840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075051.808401:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075051.808405:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075051.808408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612467840:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769075052.063821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075053.087819:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075054.111782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075054.239792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075054.239804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075054.239810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075054.239822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2042 00000100:00080000:0.0:1769075054.239836:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075054.239846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075054.239895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612468352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075054.239923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075054.239927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075054.239929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075054.239932:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2042 00000100:00080000:0.0:1769075054.239936:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075054.239939:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075054.239949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075054.239960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612468480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075054.239968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075054.239972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769075054.239972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612468352:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075054.239974:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075054.239982:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2042 00000100:00080000:0.0:1769075054.239986:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075054.239988:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075054.239992:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075054.239999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612468608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075054.240011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075054.240013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075054.240015:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075054.240017:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2042 00000100:00080000:0.0:1769075054.240020:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075054.240022:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075054.240025:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075054.240032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612468736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075054.240083:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075054.240122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff8b1006025f80 x1855007612468352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075054.240137:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612468352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075054.240147:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075054.240150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075054.240154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612468352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075054.240177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075054.240195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075054.240204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff8b1006024a80 x1855007612468480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075054.240213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612468480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075054.240219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075054.240221:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075054.240224:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075054.240227:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075054.240239:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075054.240247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff8b1006026d80 x1855007612468608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075054.240260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612468608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075054.240269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075054.240273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075054.240276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075054.240283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075054.240305:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075054.240319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff8b1006027b80 x1855007612468736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075054.240332:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612468736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075054.240360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075054.240363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075054.240365:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468736:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075055.135794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075056.159763:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075057.183817:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075058.207790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075059.231779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075059.359797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075059.359805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075059.359809:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075059.359817:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2047 00000100:00080000:1.0:1769075059.359826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075059.359832:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075059.359862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612468864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075059.359884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075059.359887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075059.359888:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075059.359895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2047 00000100:00080000:1.0:1769075059.359897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075059.359900:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075059.359907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075059.359917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612468992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075059.359929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075059.359933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075059.359935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075059.359939:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2047 00000100:00080000:1.0:1769075059.359943:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075059.359949:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075059.359967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075059.359977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612468864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075059.359989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612469120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075059.360003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075059.360007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075059.360009:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075059.360012:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2047 00000100:00080000:1.0:1769075059.360016:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075059.360018:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075059.360022:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075059.360035:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612469248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075059.360070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075059.360123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075059] req@ffff8b1006135c00 x1855007612468864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075059.360147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612468864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075059.360161:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075059.360166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075059.360171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612468864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075059.360181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075059.360209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075059.360225:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075059] req@ffff8b1006137b80 x1855007612468992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075059.360235:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612468992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075059.360243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075059.360247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075059.360252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612468992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075059.360287:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075059.360315:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075059.360330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075059] req@ffff8b1006135880 x1855007612469120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075059.360343:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612469120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075059.360349:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075059.360352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075059.360355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075059.360361:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075059.360385:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075059.360398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075059] req@ffff8b1006135500 x1855007612469248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075059.360413:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612469248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075059.360422:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075059.360426:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075059.360428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469248:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075060.255825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075061.279785:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075062.303803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075063.327810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075064.351819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075064.479782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075064.479794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075064.479800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075064.479811:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2052 00000100:00080000:1.0:1769075064.479824:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075064.479834:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075064.479894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612469376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075064.479924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075064.479927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075064.479929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075064.479934:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2052 00000100:00080000:1.0:1769075064.479938:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075064.479946:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075064.479957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075064.479972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612469376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075064.479975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612469504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075064.479999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075064.480003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075064.480005:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075064.480018:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2052 00000100:00080000:1.0:1769075064.480023:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075064.480028:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075064.480033:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075064.480047:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612469632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075064.480061:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075064.480072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075064.480076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075064.480078:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075064.480084:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2052 00000100:00080000:1.0:1769075064.480091:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075064.480095:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075064.480101:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075064.480111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff8b1006137100 x1855007612469376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075064.480115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612469760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075064.480197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612469376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.480208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.480213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075064.480219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612469376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075064.480240:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469504:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075064.480257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075064.480266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff8b1006134700 x1855007612469504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.480275:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612469504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.480280:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.480282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075064.480285:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075064.480289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075064.480303:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075064.480309:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff8b1006136d80 x1855007612469632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.480317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612469632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.480323:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.480325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075064.480327:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075064.480333:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075064.480355:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075064.480370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff8b1006136680 x1855007612469760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.480385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612469760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.480392:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.480395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075064.480398:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612469760:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075065.375778:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075066.399760:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075067.423828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075068.447840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075069.471771:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075069.599743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075069.599757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075069.599762:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075069.599776:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2057 00000100:00080000:2.0:1769075069.599803:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075069.599813:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075069.599859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612469888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075069.599889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075069.599892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075069.599894:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075069.599897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2057 00000100:00080000:2.0:1769075069.599901:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075069.599904:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075069.599915:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075069.599925:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612470016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075069.599933:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612469888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075069.599936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075069.599941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075069.599942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075069.599946:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2057 00000100:00080000:2.0:1769075069.599948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075069.599951:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075069.599954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075069.599965:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612470144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075069.599972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075069.599975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075069.599976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075069.599979:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2057 00000100:00080000:2.0:1769075069.599982:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075069.599985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075069.599988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075069.599995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612470272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075069.600050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075069.600078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075069.600094:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470272:192.168.204.102@tcp:8:kworker.0 00000400:00000100:1.0:1769075070.240222:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.204.102:988 00000400:00000100:1.0:1769075070.240291:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075070.240319:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075070.240376:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075070.240390:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075070.240400:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769075070.240588:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075070] req@ffff8b1005d58e00 x1855007612470016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075070.240628:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612470016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075070.240700:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075070.240707:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075070.240714:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075070.240728:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075070] req@ffff8b1005d5bb80 x1855007612470144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075070.240752:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612470144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075070.240762:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075070.240767:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075070.240786:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075070.240796:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075070] req@ffff8b1005d59880 x1855007612470272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075070.240811:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612470272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075070.240820:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075070.240825:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075070.240828:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075070.240836:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075070] req@ffff8b1005d59500 x1855007612469888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075070.240852:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612469888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075070.240862:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075070.240867:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075070.240871:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612469888:192.168.204.102@tcp:250:kworker.0 00010000:00010000:3.0:1769075070.495761:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075071.519739:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075072.543971:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075073.567767:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075074.591790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075074.719791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.719800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075074.719804:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.719812:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2062 00000100:00080000:1.0:1769075074.719821:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.719828:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075074.719865:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612470400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.719895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.719898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075074.719900:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.719903:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2062 00000100:00080000:1.0:1769075074.719906:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.719910:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075074.719917:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075074.719930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612470528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.719939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.719944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075074.719946:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075074.719947:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612470400:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075074.719951:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2062 00000100:00080000:1.0:1769075074.719957:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.719961:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075074.719964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075074.719972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612470656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.719979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.719981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075074.719982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.719985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2062 00000100:00080000:1.0:1769075074.719987:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.719989:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075074.719993:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075074.720004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612470784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075074.720021:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075074.720061:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff8b1006137480 x1855007612470400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.720081:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612470400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.720092:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.720097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075074.720101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612470400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075074.720124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470528:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075074.720147:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075074.720162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff8b1006134a80 x1855007612470528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.720176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612470528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.720185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.720189:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075074.720193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075074.720200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075074.720225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075074.720248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff8b1006134e00 x1855007612470656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.720264:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612470656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.720273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.720276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075074.720280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075074.720288:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075074.720311:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075074.720323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff8b1006136680 x1855007612470784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.720337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612470784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.720347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.720350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075074.720354:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612470784:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075075.615763:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075076.639839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075077.663807:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075078.687831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075079.711771:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075079.839754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075079.839766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075079.839770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075079.839780:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2067 00000100:00080000:1.0:1769075079.839795:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075079.839801:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075079.839847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612470912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075079.839883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075079.839886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075079.839887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075079.839890:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2067 00000100:00080000:1.0:1769075079.839893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075079.839896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075079.839905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075079.839916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612471040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075079.839928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075079.839932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075079.839934:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075079.839941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612470912:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075079.839943:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2067 00000100:00080000:1.0:1769075079.839950:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075079.839956:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075079.839962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075079.839976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612471168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075079.839987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075079.839991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075079.839993:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075079.839996:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2067 00000100:00080000:1.0:1769075079.840020:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075079.840023:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075079.840027:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769075079.840034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075079.840039:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612471296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075079.840088:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075079] req@ffff8b1006136d80 x1855007612470912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075079.840111:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612470912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075079.840124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075079.840129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075079.840134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612470912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075079.840165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075079.840192:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075079.840206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075079] req@ffff8b1006134700 x1855007612471040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075079.840220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612471040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075079.840229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075079.840233:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075079.840237:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075079.840244:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075079.840267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075079.840281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075079] req@ffff8b1006137100 x1855007612471168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075079.840294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612471168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075079.840310:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075079.840314:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075079.840318:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075079.840324:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075079.840346:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075079.840358:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075079] req@ffff8b1006135500 x1855007612471296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075079.840372:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612471296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075079.840381:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075079.840385:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075079.840388:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471296:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075080.735781:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075081.759776:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075082.783805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075083.807771:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075084.831813:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075084.959707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075084.959719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075084.959724:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075084.959735:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2073 00000100:00080000:1.0:1769075084.959748:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075084.959755:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075084.959806:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612471424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075084.959842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075084.959847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075084.959849:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075084.959854:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2073 00000100:00080000:1.0:1769075084.959858:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075084.959863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075084.959876:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075084.959895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612471552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075084.959896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612471424:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075084.959915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075084.959918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075084.959921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075084.959926:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2073 00000100:00080000:1.0:1769075084.959931:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075084.959936:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075084.959942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075084.959956:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612471680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075084.959965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075084.959968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075084.959970:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075084.959973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2073 00000800:00000100:0.0:1769075084.959978:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075084.959980:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075084.959985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075084.959990:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075084.960009:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612471808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075084.960019:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff8b1006135880 x1855007612471424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.960037:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612471424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.960047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.960051:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075084.960054:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612471424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075084.960077:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075084.960093:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075084.960103:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff8b1006137b80 x1855007612471552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.960112:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612471552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.960118:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.960120:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075084.960123:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075084.960127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075084.960138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075084.960144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff8b1006135c00 x1855007612471680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.960155:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612471680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.960164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.960167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075084.960171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075084.960177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075084.960200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075084.960213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff8b1006136300 x1855007612471808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.960226:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612471808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.960232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.960235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075084.960237:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612471808:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075085.855823:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075086.879798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075087.903826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075088.927832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075089.951764:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075090.079778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075090.079788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075090.079793:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075090.079814:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2078 00000100:00080000:2.0:1769075090.079824:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075090.079830:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075090.079865:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612471936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075090.079889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075090.079892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075090.079894:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075090.079897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2078 00000100:00080000:2.0:1769075090.079900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075090.079903:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075090.079910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075090.079921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612472064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075090.079929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075090.079934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075090.079936:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075090.079939:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2078 00000100:00080000:2.0:1769075090.079941:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075090.079946:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612471936:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075090.079947:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075090.079954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075090.079963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612472192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075090.079971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075090.079974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075090.079975:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075090.079978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2078 00000100:00080000:2.0:1769075090.079981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075090.079983:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075090.079987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075090.079995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612472320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075090.080013:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075090.080045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff8b1005d5aa00 x1855007612471936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.080060:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612471936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.080069:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.080072:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075090.080076:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612471936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075090.080082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472064:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075090.080098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075090.080107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff8b1005d5b100 x1855007612472064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.080117:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612472064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.080123:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.080125:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075090.080128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075090.080167:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075090.080185:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075090.080195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff8b1005d58380 x1855007612472192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.080206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612472192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.080213:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.080216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075090.080219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075090.080224:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075090.080238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075090.080246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff8b1005d5ad80 x1855007612472320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.080257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612472320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.080264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.080267:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075090.080270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472320:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075090.975746:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075091.999835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075093.023768:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075094.047799:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075095.071790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075095.199832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075095.199845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075095.199850:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075095.199861:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2083 00000100:00080000:1.0:1769075095.199874:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075095.199881:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075095.199926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612472448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075095.199955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075095.199959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075095.199960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075095.199964:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2083 00000100:00080000:1.0:1769075095.199966:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075095.199969:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075095.199979:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075095.199989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612472576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075095.199997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075095.200000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075095.200004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075095.200006:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612472448:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075095.200009:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2083 00000100:00080000:1.0:1769075095.200013:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075095.200018:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075095.200022:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075095.200030:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612472704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075095.200037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075095.200041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075095.200043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075095.200045:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2083 00000100:00080000:1.0:1769075095.200048:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075095.200050:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075095.200054:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075095.200063:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612472832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075095.200100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075095.200138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff8b1006136a00 x1855007612472448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.200154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612472448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.200164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.200168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075095.200171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612472448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075095.200194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075095.200211:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075095.200220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff8b1006135180 x1855007612472576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.200230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612472576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.200235:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.200239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075095.200241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075095.200246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075095.200274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075095.200282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff8b1006137800 x1855007612472704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.200291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612472704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.200298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.200300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075095.200302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075095.200306:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075095.200319:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075095.200327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff8b1006136300 x1855007612472832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.200336:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612472832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.200342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.200344:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075095.200347:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612472832:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075096.095795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075097.119800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075098.143857:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075099.167788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075100.191772:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075100.319736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075100.319749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075100.319754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075100.319765:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2088 00000100:00080000:1.0:1769075100.319777:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075100.319784:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075100.319828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612472960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075100.319855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075100.319858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075100.319860:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075100.319863:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2088 00000100:00080000:1.0:1769075100.319866:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075100.319869:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075100.319878:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075100.319889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612473088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075100.319900:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612472960:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075100.319901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075100.319909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075100.319912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075100.319916:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2088 00000100:00080000:1.0:1769075100.319918:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075100.319921:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075100.319924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075100.319933:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612473216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075100.319938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075100.319941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075100.319942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075100.319945:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2088 00000100:00080000:1.0:1769075100.319948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075100.319950:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075100.319953:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075100.319960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612473344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075100.319983:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075100.320021:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff8b1006135c00 x1855007612472960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075100.320044:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612472960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075100.320056:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075100.320062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075100.320068:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612472960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075100.320099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473088:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075100.320123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075100.320137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff8b1006137b80 x1855007612473088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075100.320150:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612473088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075100.320159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075100.320164:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075100.320168:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075100.320176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075100.320200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075100.320213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff8b1006135880 x1855007612473216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075100.320227:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612473216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075100.320236:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075100.320239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075100.320242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075100.320247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075100.320265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075100.320289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff8b1006135500 x1855007612473344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075100.320299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612473344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075100.320308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075100.320311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075100.320315:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473344:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075101.215838:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075102.239806:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075103.263932:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075104.287873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075105.311787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075105.439759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075105.439771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075105.439776:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075105.439786:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2093 00000100:00080000:2.0:1769075105.439799:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075105.439808:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075105.439862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612473472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075105.439891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075105.439894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075105.439896:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075105.439899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2093 00000100:00080000:2.0:1769075105.439902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075105.439907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075105.439917:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075105.439927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612473472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075105.439932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612473600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075105.439942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075105.439945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075105.439946:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075105.439949:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2093 00000100:00080000:2.0:1769075105.439951:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075105.439954:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075105.439957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075105.439965:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612473728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075105.439971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075105.439973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075105.439975:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075105.439978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2093 00000100:00080000:2.0:1769075105.439982:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075105.439986:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075105.439991:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075105.440005:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612473856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075105.440044:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075105.440066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075105.440083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473856:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075106.335770:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075107.104061:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769075107.104122:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075107.104147:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075107.104222:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075107.104235:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075107.104246:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769075107.104343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075107] req@ffff8b1005d58380 x1855007612473600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075107.104377:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612473600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075107.104393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075107.104401:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075107.104406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075107.104417:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075107] req@ffff8b1005d5b100 x1855007612473728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075107.104433:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612473728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075107.104442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075107.104454:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075107.104459:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075107.104467:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075107] req@ffff8b1005d5aa00 x1855007612473856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075107.104481:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612473856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075107.104489:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075107.104494:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075107.104497:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612473856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075107.104505:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075107] req@ffff8b1005d5ad80 x1855007612473472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075107.104538:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612473472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075107.104549:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075107.104553:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075107.104557:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612473472:192.168.204.102@tcp:250:kworker.0 00010000:00010000:3.0:1769075107.359812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075108.383916:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075109.407807:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075110.431912:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075110.559754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075110.559770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075110.559776:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075110.559786:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2098 00000100:00080000:2.0:1769075110.559799:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075110.559806:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075110.559852:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612473984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075110.559881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075110.559884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075110.559886:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075110.559889:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2098 00000100:00080000:2.0:1769075110.559892:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075110.559894:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075110.559903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075110.559923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612474112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075110.559930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075110.559935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075110.559936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075110.559938:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2098 00000100:00080000:2.0:1769075110.559940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075110.559942:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075110.559946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075110.559952:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612474240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075110.559958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075110.559961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075110.559963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075110.559966:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2098 00000100:00080000:2.0:1769075110.559968:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075110.559970:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075110.559973:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075110.559979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612474368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075110.560008:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612473984:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769075110.560110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075110.560146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff8b1005d59500 x1855007612473984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075110.560161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612473984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075110.560170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075110.560173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075110.560176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612473984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075110.560182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075110.560198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075110.560206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff8b1005d59880 x1855007612474112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075110.560215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612474112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075110.560220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075110.560222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075110.560225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075110.560229:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075110.560254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075110.560266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff8b1005d5bb80 x1855007612474240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075110.560278:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612474240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075110.560287:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075110.560290:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075110.560294:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075110.560301:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075110.560322:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075110.560333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff8b1005d58e00 x1855007612474368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075110.560349:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612474368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075110.560355:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075110.560358:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075110.560360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474368:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075111.455761:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075112.479751:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075113.503777:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075114.527774:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075115.551819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075115.679739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075115.679751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075115.679756:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075115.679766:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2103 00000100:00080000:0.0:1769075115.679778:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075115.679785:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075115.679830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612474496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075115.679858:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075115.679862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075115.679863:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075115.679866:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2103 00000100:00080000:0.0:1769075115.679879:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075115.679882:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075115.679892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075115.679901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612474624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075115.679910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075115.679915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075115.679917:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075115.679917:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612474496:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075115.679925:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2103 00000100:00080000:0.0:1769075115.679928:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075115.679931:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075115.679934:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075115.679942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612474752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075115.679949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075115.679952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075115.679953:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075115.679956:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2103 00000100:00080000:0.0:1769075115.679958:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075115.679961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075115.679964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075115.679970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612474880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075115.680005:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075115.680055:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075115] req@ffff8b1006024700 x1855007612474496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075115.680072:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612474496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075115.680082:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075115.680086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075115.680089:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612474496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075115.680111:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474624:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075115.680129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075115.680138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075115] req@ffff8b1006024380 x1855007612474624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075115.680147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612474624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075115.680152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075115.680155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075115.680157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075115.680161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075115.680172:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075115.680178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075115] req@ffff8b1006027480 x1855007612474752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075115.680186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612474752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075115.680192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075115.680194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075115.680196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075115.680201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075115.680212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075115.680218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075115] req@ffff8b1006024e00 x1855007612474880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075115.680226:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612474880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075115.680231:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075115.680233:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075115.680235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612474880:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075116.575762:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075117.599759:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075118.623824:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075119.647798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075120.671776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075120.799791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.799804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075120.799810:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.799822:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2108 00000100:00080000:1.0:1769075120.799835:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.799843:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075120.799889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612475008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.799915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.799918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075120.799920:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.799936:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2108 00000100:00080000:1.0:1769075120.799940:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.799943:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075120.799953:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075120.799974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612475136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075120.799982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612475008:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075120.799985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.799992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075120.799993:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.799997:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2108 00000100:00080000:1.0:1769075120.799999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.800004:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075120.800008:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075120.800017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612475264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.800024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.800027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075120.800028:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.800031:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2108 00000100:00080000:1.0:1769075120.800033:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.800036:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075120.800038:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075120.800045:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612475392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075120.800079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075120.800116:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff8b1006137100 x1855007612475008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075120.800145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612475008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075120.800154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075120.800158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075120.800161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612475008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075120.800184:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075120.800202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075120.800211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff8b1006134700 x1855007612475136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075120.800220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612475136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075120.800225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075120.800227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075120.800229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075120.800233:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075120.800245:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075120.800251:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff8b1006136d80 x1855007612475264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075120.800259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612475264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075120.800264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075120.800266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075120.800268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075120.800272:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075120.800282:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075120.800289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff8b1006136680 x1855007612475392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075120.800301:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612475392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075120.800307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075120.800309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075120.800310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475392:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075121.695785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075122.719880:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075123.743819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075124.767892:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075125.791826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075125.919757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075125.919770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075125.919775:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075125.919785:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2114 00000100:00080000:1.0:1769075125.919798:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075125.919805:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075125.919875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612475520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075125.919903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075125.919906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075125.919909:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075125.919912:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2114 00000100:00080000:1.0:1769075125.919915:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075125.919918:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075125.919928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075125.919938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612475648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075125.919945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075125.919948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075125.919950:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075125.919953:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2114 00000100:00080000:1.0:1769075125.919955:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075125.919958:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075125.919962:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075125.919968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612475776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075125.919975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075125.919977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075125.919979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075125.919983:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2114 00000100:00080000:1.0:1769075125.919986:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075125.919988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075125.919992:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075125.920002:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612475904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075125.920005:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612475520:192.168.204.102@tcp:250:kworker.0 00000800:00000100:3.0:1769075125.920085:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075125.920121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff8b1006134e00 x1855007612475520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075125.920135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612475520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075125.920143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075125.920146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075125.920150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612475520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075125.920156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075125.920171:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075125.920179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff8b1006134a80 x1855007612475648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075125.920192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612475648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075125.920197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075125.920199:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075125.920201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075125.920205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075125.920216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075125.920222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff8b1006137480 x1855007612475776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075125.920230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612475776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075125.920235:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075125.920237:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075125.920239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075125.920258:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075125.920269:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075125.920275:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff8b1006136680 x1855007612475904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075125.920282:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612475904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075125.920292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075125.920294:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075125.920296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612475904:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075126.815903:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075127.839839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075128.863884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075129.887901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075130.911899:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075131.039839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075131.039852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075131.039858:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075131.039869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2119 00000100:00080000:0.0:1769075131.039884:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075131.039891:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075131.039952:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612476032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075131.039981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075131.039985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075131.039987:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075131.039991:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2119 00000100:00080000:0.0:1769075131.039994:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075131.039996:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075131.040006:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075131.040038:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612476160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075131.040047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075131.040050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075131.040051:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075131.040055:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2119 00000100:00080000:0.0:1769075131.040058:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075131.040062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075131.040065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612476032:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769075131.040067:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075131.040078:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612476288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075131.040090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075131.040093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075131.040094:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075131.040114:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2119 00000100:00080000:0.0:1769075131.040119:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075131.040123:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075131.040128:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075131.040141:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612476416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075131.040218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075131.040293:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075131/real 1769075131] req@ffff8b1006024e00 x1855007612476032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075131.040311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612476032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075131.040347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075131.040352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075131.040356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612476032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075131.040365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476160:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075131.040390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075131.040403:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075131/real 1769075131] req@ffff8b1006027480 x1855007612476160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075131.040417:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612476160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075131.040425:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075131.040428:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075131.040431:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075131.040457:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075131.040473:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075131.040480:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075131/real 1769075131] req@ffff8b1006024380 x1855007612476288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075131.040488:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612476288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075131.040494:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075131.040496:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075131.040498:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075131.040503:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075131.040540:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075131.040556:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075131/real 1769075131] req@ffff8b1006024700 x1855007612476416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075131.040573:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612476416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075131.040600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075131.040614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075131.040617:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476416:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075131.935776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075132.959881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075133.983966:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075135.007821:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075136.032001:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075136.159857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075136.159871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075136.159877:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075136.159890:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2124 00000100:00080000:0.0:1769075136.159906:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075136.159917:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075136.159979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612476544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075136.160018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075136.160022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075136.160024:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075136.160028:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2124 00000100:00080000:0.0:1769075136.160033:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075136.160039:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075136.160051:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075136.160066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612476672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075136.160075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612476544:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075136.160080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075136.160085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075136.160087:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075136.160091:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2124 00000100:00080000:0.0:1769075136.160095:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075136.160098:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075136.160103:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075136.160116:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612476800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075136.160124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075136.160128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075136.160129:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075136.160132:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2124 00000100:00080000:0.0:1769075136.160136:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075136.160139:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075136.160143:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075136.160151:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612476928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075136.160177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075136.160227:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff8b1006027b80 x1855007612476544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.160250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612476544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.160261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.160268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075136.160276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612476544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075136.160313:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075136.160356:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075136.160378:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff8b1006026d80 x1855007612476672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.160395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612476672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.160406:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.160410:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075136.160424:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075136.160433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075136.160455:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075136.160468:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff8b1006024a80 x1855007612476800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.160600:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612476800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.160609:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.160612:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075136.160615:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075136.160624:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075136.160746:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075136.160956:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff8b1006025f80 x1855007612476928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.160971:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612476928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.160980:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.160984:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075136.161039:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612476928:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075137.055930:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075138.079841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075139.103789:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075140.127837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075141.151900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075141.279887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.279898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075141.279903:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.279913:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2129 00000100:00080000:1.0:1769075141.279925:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.279932:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075141.279974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612477056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.280005:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.280009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075141.280011:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.280015:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2129 00000100:00080000:1.0:1769075141.280018:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.280021:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075141.280030:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075141.280044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612477184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.280055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.280059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075141.280062:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.280066:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2129 00000100:00080000:1.0:1769075141.280069:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.280073:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075141.280080:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075141.280085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612477056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075141.280096:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612477312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.280108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.280112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075141.280114:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.280120:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2129 00000100:00080000:1.0:1769075141.280143:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.280148:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075141.280153:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075141.280164:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612477440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075141.280202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075141.280262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff8b1006136d80 x1855007612477056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.280285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612477056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.280295:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.280299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075141.280303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612477056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075141.280310:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477184:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075141.280326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075141.280335:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff8b1006134700 x1855007612477184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.280343:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612477184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.280359:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.280361:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075141.280363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075141.280389:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075141.280401:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075141.280407:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff8b1006137100 x1855007612477312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.280415:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612477312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.280425:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.280427:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075141.280429:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075141.280433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075141.280443:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075141.280450:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff8b1006135500 x1855007612477440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.280457:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612477440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.280462:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.280465:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075141.280466:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477440:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075142.175853:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075143.199841:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075144.223991:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075145.247799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075146.271919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075146.399768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.399788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075146.399795:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.399812:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2134 00000100:00080000:2.0:1769075146.399842:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.399854:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075146.399921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612477568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.399957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.399962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075146.399964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.399979:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2134 00000100:00080000:2.0:1769075146.399983:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.399988:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075146.400000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075146.400012:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612477696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.400022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.400026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075146.400031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075146.400044:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612477568:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075146.400048:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2134 00000100:00080000:2.0:1769075146.400056:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.400060:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075146.400065:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075146.400078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612477824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.400093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.400096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075146.400097:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.400101:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2134 00000100:00080000:2.0:1769075146.400123:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.400129:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075146.400135:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075146.400144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075146.400154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612477952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075146.400216:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff8b1005d59180 x1855007612477568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.400247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612477568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.400271:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.400276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075146.400281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612477568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075146.400324:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075146.400371:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075146.400397:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff8b1005d5a300 x1855007612477696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.400431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612477696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.400442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.400447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075146.400451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075146.400458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075146.400518:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075146.400530:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff8b1005d5b800 x1855007612477824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.400541:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612477824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.400548:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.400551:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075146.400554:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075146.400560:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075146.400589:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075146.400599:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff8b1005d5b480 x1855007612477952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.400610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612477952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.400617:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.400621:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075146.400624:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612477952:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075147.295811:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075148.319809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075149.343811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075150.367818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075151.391877:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075151.519752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075151.519767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075151.519772:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075151.519784:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2139 00000100:00080000:1.0:1769075151.519797:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075151.519805:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075151.519850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612478080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075151.519880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075151.519884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075151.519887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075151.519892:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2139 00000100:00080000:1.0:1769075151.519896:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075151.519901:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075151.519914:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075151.519932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612478208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075151.519933:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612478080:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075151.519957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075151.519961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075151.519963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075151.519966:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2139 00000100:00080000:1.0:1769075151.519970:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075151.519973:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075151.519979:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075151.519990:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612478336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075151.519997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075151.520000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075151.520002:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075151.520005:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2139 00000100:00080000:1.0:1769075151.520008:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075151.520010:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075151.520014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075151.520019:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075151.520032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612478464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075151.520063:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff8b1006135880 x1855007612478080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.520082:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612478080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.520091:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.520095:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075151.520099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612478080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075151.520122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075151.520141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075151.520151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff8b1006137b80 x1855007612478208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.520160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612478208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.520166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.520168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075151.520179:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075151.520185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075151.520199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075151.520207:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff8b1006135c00 x1855007612478336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.520215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612478336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.520221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.520224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075151.520226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075151.520230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075151.520240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075151.520247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff8b1006136300 x1855007612478464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.520260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612478464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.520265:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.520267:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075151.520269:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478464:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075152.415848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075153.439925:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075154.463785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075155.487877:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075156.511796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075156.639767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075156.639778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075156.639784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075156.639794:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2144 00000100:00080000:2.0:1769075156.639808:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075156.639815:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075156.639871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612478592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075156.639900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075156.639904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075156.639907:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075156.639915:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2144 00000100:00080000:2.0:1769075156.639919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075156.639923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075156.639936:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075156.639952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612478720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075156.639957:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612478592:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075156.639965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075156.639969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075156.639971:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075156.639975:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2144 00000100:00080000:2.0:1769075156.639979:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075156.639983:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075156.639989:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075156.639998:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59c00 x1855007612478848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075156.640008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075156.640011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075156.640013:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075156.640016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2144 00000100:00080000:2.0:1769075156.640039:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075156.640044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075156.640048:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075156.640054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075156.640060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a680 x1855007612478976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075156.640101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff8b1005d58a80 x1855007612478592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.640124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612478592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.640153:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.640157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075156.640161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612478592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075156.640209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478720:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075156.640243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075156.640262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff8b1005d58000 x1855007612478720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.640311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612478720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.640320:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.640339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075156.640343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075156.640366:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075156.640394:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075156.640404:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff8b1005d59c00 x1855007612478848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.640413:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59c00 x1855007612478848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.640419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.640422:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075156.640424:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075156.640428:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075156.640449:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075156.640457:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff8b1005d5a680 x1855007612478976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.640466:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a680 x1855007612478976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.640477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.640480:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075156.640482:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612478976:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075157.535970:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075158.559836:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075159.583889:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075160.607864:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075161.631892:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075161.759711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075161.759725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075161.759730:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075161.759741:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2149 00000100:00080000:2.0:1769075161.759755:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075161.759764:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075161.759812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a680 x1855007612479104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075161.759848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075161.759852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075161.759854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075161.759858:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2149 00000100:00080000:2.0:1769075161.759860:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075161.759863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075161.759873:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075161.759884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59c00 x1855007612479232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075161.759892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075161.759895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075161.759897:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075161.759900:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2149 00000100:00080000:2.0:1769075161.759903:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075161.759908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075161.759912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075161.759920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612479360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075161.759930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075161.759932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612479104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075161.759934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075161.759938:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075161.759944:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2149 00000100:00080000:2.0:1769075161.759947:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075161.759949:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075161.759953:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075161.759963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612479488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075161.760041:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075161.760095:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff8b1005d5a680 x1855007612479104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.760140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a680 x1855007612479104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.760156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.760163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075161.760168:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612479104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075161.760177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075161.760219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075161.760231:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff8b1005d59c00 x1855007612479232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.760261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59c00 x1855007612479232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.760268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.760271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075161.760274:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075161.760302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075161.760321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075161.760329:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff8b1005d58000 x1855007612479360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.760338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612479360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.760345:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.760347:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075161.760349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075161.760353:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075161.760366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075161.760373:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff8b1005d58a80 x1855007612479488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.760382:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612479488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.760388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.760390:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075161.760392:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479488:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075162.655843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075163.679951:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075164.703875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075165.727967:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075166.751842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075166.879819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075166.879834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075166.879840:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075166.879854:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2154 00000100:00080000:2.0:1769075166.879872:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075166.879883:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075166.879943:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612479616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075166.879976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075166.879981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075166.879984:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075166.879989:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2154 00000100:00080000:2.0:1769075166.879993:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075166.879997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075166.880009:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075166.880023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612479744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075166.880036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075166.880040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075166.880042:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075166.880046:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2154 00000100:00080000:2.0:1769075166.880050:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075166.880053:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075166.880059:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075166.880065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612479616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075166.880073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612479872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075166.880086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075166.880089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075166.880091:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075166.880095:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2154 00000100:00080000:2.0:1769075166.880100:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075166.880103:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075166.880108:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075166.880122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612480000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075166.880257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075166.880304:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075166.880329:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480000:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075167.775902:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075168.799852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075169.823837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075169.952033:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075169.952087:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075169.952137:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075169.952204:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075169.952217:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075169.952230:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075169.952282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075169] req@ffff8b1005d5a300 x1855007612479872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075169.952340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612479872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075169.952358:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075169.952367:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075169.952375:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075169.952390:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075169] req@ffff8b1005d59180 x1855007612480000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075169.952414:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612480000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075169.952429:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075169.952436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075169.952443:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075169.952461:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075169] req@ffff8b1005d5b480 x1855007612479616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075169.952484:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612479616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075169.952498:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075169.952504:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075169.952510:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612479616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075169.952522:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075169] req@ffff8b1005d5b800 x1855007612479744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075169.952544:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612479744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075169.952557:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075169.952563:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075169.952569:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612479744:192.168.204.102@tcp:38:kworker.0 00010000:00010000:3.0:1769075170.847889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075171.871828:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075171.999751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.999770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075171.999777:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.999792:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2160 00000100:00080000:1.0:1769075171.999822:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.999833:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075171.999886:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612480128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075171.999935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.999941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075171.999943:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.999949:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2160 00000100:00080000:1.0:1769075171.999954:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.999959:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075171.999972:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075171.999988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612480256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075172.000002:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612480128:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075172.000005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075172.000016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075172.000019:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075172.000026:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2160 00000100:00080000:1.0:1769075172.000031:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075172.000038:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075172.000045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075172.000061:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612480384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075172.000086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075172.000091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075172.000093:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075172.000098:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2160 00000100:00080000:1.0:1769075172.000104:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075172.000109:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075172.000116:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075172.000130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612480512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075172.000146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075172.000194:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075172] req@ffff8b1006137800 x1855007612480128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075172.000216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612480128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075172.000229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075172.000234:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075172.000238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612480128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075172.000268:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075172.000294:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075172.000307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075172] req@ffff8b1006135180 x1855007612480256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075172.000330:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612480256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075172.000339:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075172.000343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075172.000346:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075172.000353:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075172.000375:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075172.000387:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075172] req@ffff8b1006136a00 x1855007612480384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075172.000400:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612480384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075172.000409:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075172.000413:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075172.000416:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075172.000423:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075172.000445:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075172.000458:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075172] req@ffff8b1006136300 x1855007612480512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075172.000469:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612480512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075172.000483:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075172.000486:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075172.000490:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480512:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075172.895992:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075173.919826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075174.943939:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075175.967851:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075176.991792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075177.119726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.119738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075177.119742:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.119753:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2165 00000100:00080000:1.0:1769075177.119766:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.119773:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075177.119816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612480640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.119855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.119858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075177.119861:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.119864:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2165 00000100:00080000:1.0:1769075177.119867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.119870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075177.119879:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075177.119888:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612480768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.119895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.119908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075177.119909:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.119912:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2165 00000100:00080000:1.0:1769075177.119913:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.119916:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075177.119919:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075177.119926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612480896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.119931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.119934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075177.119936:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.119939:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2165 00000100:00080000:1.0:1769075177.119941:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.119943:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075177.119948:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075177.119949:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612480640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075177.119960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612481024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075177.120044:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075177.120083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff8b1006135c00 x1855007612480640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.120099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612480640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.120108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.120112:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075177.120116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612480640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075177.120121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075177.120137:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075177.120145:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff8b1006137b80 x1855007612480768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.120154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612480768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.120159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.120161:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075177.120164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075177.120169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075177.120181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075177.120187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff8b1006135880 x1855007612480896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.120194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612480896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.120200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.120201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075177.120203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612480896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075177.120226:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075177.120238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075177.120245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff8b1006135500 x1855007612481024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.120253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612481024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.120258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.120259:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075177.120261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481024:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075178.015797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075179.039926:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075180.063869:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075181.087815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075182.111846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075182.239733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075182.239747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075182.239752:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075182.239765:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2170 00000100:00080000:1.0:1769075182.239788:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075182.239797:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075182.239849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612481152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075182.239878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075182.239882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075182.239884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075182.239888:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2170 00000100:00080000:1.0:1769075182.239908:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075182.239912:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075182.239923:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075182.239935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612481280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075182.239944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075182.239948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075182.239950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075182.239954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2170 00000100:00080000:1.0:1769075182.239958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075182.239962:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075182.239967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075182.239975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612481408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075182.239983:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612481152:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075182.239986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075182.239995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075182.239997:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075182.240000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2170 00000100:00080000:1.0:1769075182.240003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075182.240006:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075182.240010:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075182.240021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612481536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075182.240080:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075182.240121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075182] req@ffff8b1006137100 x1855007612481152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075182.240139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612481152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075182.240151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075182.240155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075182.240161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612481152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075182.240170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481280:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075182.240199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075182.240216:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075182] req@ffff8b1006134700 x1855007612481280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075182.240234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612481280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075182.240245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075182.240250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075182.240254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075182.240286:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075182.240308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075182.240318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075182] req@ffff8b1006136d80 x1855007612481408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075182.240337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612481408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075182.240347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075182.240351:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075182.240356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075182.240364:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075182.240390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075182.240406:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075182] req@ffff8b1006136680 x1855007612481536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075182.240421:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612481536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075182.240429:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075182.240431:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075182.240434:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481536:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075183.135835:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075184.159846:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075185.183805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075186.207823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075187.231801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075187.359724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075187.359733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075187.359737:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075187.359745:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2175 00000100:00080000:0.0:1769075187.359755:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075187.359761:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075187.359792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612481664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075187.359825:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075187.359828:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075187.359830:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075187.359845:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2175 00000100:00080000:0.0:1769075187.359848:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075187.359851:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075187.359860:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075187.359871:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612481792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075187.359879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612481664:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075187.359882:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075187.359887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075187.359891:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075187.359895:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2175 00000100:00080000:0.0:1769075187.359897:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075187.359899:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075187.359903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075187.359918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612481920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075187.359924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075187.359927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075187.359928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075187.359930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2175 00000100:00080000:0.0:1769075187.359933:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075187.359936:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075187.359940:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075187.359948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612482048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075187.359955:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075187.359992:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff8b1006025880 x1855007612481664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075187.360007:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612481664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075187.360020:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075187.360023:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075187.360026:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612481664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075187.360044:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075187.360062:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075187.360070:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff8b1006026a00 x1855007612481792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075187.360078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612481792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075187.360084:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075187.360086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075187.360089:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075187.360092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075187.360103:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075187.360109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff8b1006026300 x1855007612481920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075187.360117:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612481920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075187.360122:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075187.360124:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075187.360126:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612481920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075187.360130:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075187.360140:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075187.360146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff8b1006024000 x1855007612482048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075187.360153:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612482048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075187.360158:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075187.360160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075187.360162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482048:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075188.255843:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075189.279872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075190.303999:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075191.327901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075192.351893:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075192.479807:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075192.479817:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075192.479822:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075192.479830:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2180 00000100:00080000:1.0:1769075192.479840:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075192.479847:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075192.479882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612482176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075192.479907:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075192.479910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075192.479912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075192.479916:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2180 00000100:00080000:1.0:1769075192.479919:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075192.479922:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075192.479930:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075192.479942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612482304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075192.479952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075192.479956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075192.479958:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075192.479964:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2180 00000100:00100000:2.0:1769075192.479964:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612482176:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075192.479968:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075192.479975:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075192.479981:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075192.479997:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612482432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075192.480009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075192.480014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075192.480017:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075192.480022:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2180 00000100:00080000:1.0:1769075192.480028:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075192.480033:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075192.480040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075192.480052:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075192.480060:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612482560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075192.480092:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff8b1006137480 x1855007612482176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075192.480131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612482176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075192.480144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075192.480149:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075192.480154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612482176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075192.480180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482304:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075192.480209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075192.480223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff8b1006134a80 x1855007612482304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075192.480238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612482304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075192.480247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075192.480251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075192.480254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075192.480261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075192.480280:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075192.480290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff8b1006134e00 x1855007612482432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075192.480303:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612482432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075192.480311:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075192.480314:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075192.480317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075192.480323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075192.480360:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075192.480374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff8b1006136680 x1855007612482560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075192.480389:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612482560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075192.480409:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075192.480413:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075192.480418:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482560:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075193.375775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075194.399948:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075195.423923:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075196.447957:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075197.471888:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075197.599916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075197.599934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075197.599942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075197.599958:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2185 00000100:00080000:0.0:1769075197.599975:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075197.599984:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075197.600043:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612482688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075197.600096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075197.600104:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075197.600108:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075197.600117:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2185 00000100:00080000:0.0:1769075197.600125:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075197.600132:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075197.600145:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075197.600165:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612482816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075197.600184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075197.600190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075197.600192:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075197.600198:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2185 00000100:00080000:0.0:1769075197.600203:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075197.600209:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075197.600217:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075197.600241:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612482944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075197.600244:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612482688:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075197.600284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075197.600288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075197.600290:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075197.600299:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2185 00000100:00080000:0.0:1769075197.600305:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075197.600321:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075197.600327:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075197.600346:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612483072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075197.600396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075197.600456:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff8b1006026680 x1855007612482688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075197.600484:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612482688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075197.600502:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075197.600509:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075197.600514:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612482688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075197.600525:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482816:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075197.600563:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075197.600588:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff8b1006025180 x1855007612482816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075197.600613:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612482816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075197.600630:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075197.600727:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075197.600735:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075197.600779:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075197.600817:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075197.600833:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff8b1006027800 x1855007612482944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075197.600850:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612482944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075197.600862:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075197.600866:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075197.600871:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612482944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075197.600878:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075197.600900:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075197.600912:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff8b1006027100 x1855007612483072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075197.600926:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612483072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075197.600935:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075197.600939:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075197.600943:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483072:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075198.495913:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075199.519798:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075200.543849:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075201.567895:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075202.591806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075202.719736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075202.719749:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075202.719755:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075202.719766:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2190 00000100:00080000:0.0:1769075202.719780:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075202.719788:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075202.719837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612483200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075202.719866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075202.719871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075202.719872:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075202.719876:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2190 00000100:00080000:0.0:1769075202.719879:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075202.719882:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075202.719892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075202.719905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612483328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075202.719915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612483200:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075202.719921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075202.719926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075202.719929:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075202.719935:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2190 00000100:00080000:0.0:1769075202.719940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075202.719945:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075202.719952:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075202.719966:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612483456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075202.719973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075202.719976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075202.719978:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075202.719981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2190 00000100:00080000:0.0:1769075202.719988:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075202.719991:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075202.719995:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075202.720005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612483584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075202.720006:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075202.720061:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff8b1006027100 x1855007612483200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.720083:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612483200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.720102:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.720106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075202.720110:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612483200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075202.720138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075202.720160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075202.720170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff8b1006027800 x1855007612483328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.720179:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612483328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.720185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.720188:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075202.720190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075202.720194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075202.720206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075202.720213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff8b1006025180 x1855007612483456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.720220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612483456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.720229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.720231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075202.720233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075202.720237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075202.720248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075202.720254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff8b1006026680 x1855007612483584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.720261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612483584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.720265:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.720268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075202.720270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483584:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075203.615962:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075204.639844:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075205.663919:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075206.687766:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075207.711771:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075207.839737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075207.839747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075207.839752:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075207.839769:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2195 00000100:00080000:0.0:1769075207.839782:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075207.839790:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075207.839829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612483712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075207.839863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075207.839868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075207.839870:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075207.839875:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2195 00000100:00080000:0.0:1769075207.839880:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075207.839885:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075207.839896:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075207.839903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612483712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075207.839913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612483840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075207.839938:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075207.839943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075207.839946:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075207.839952:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2195 00000100:00080000:0.0:1769075207.839956:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075207.839961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075207.839969:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075207.839969:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075207.839987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612483968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075207.839999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075207.840003:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075207.840005:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075207.840012:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2195 00000100:00100000:1.0:1769075207.840013:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff8b1006024000 x1855007612483712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075207.840018:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075207.840028:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075207.840035:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075207.840039:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612483712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075207.840049:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769075207.840053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612484096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075207.840056:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075207.840063:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612483712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075207.840088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483840:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075207.840111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075207.840121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff8b1006026300 x1855007612483840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075207.840130:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612483840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075207.840135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075207.840138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075207.840140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075207.840144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075207.840158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075207.840165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff8b1006026a00 x1855007612483968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075207.840177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612483968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075207.840186:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075207.840190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075207.840194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612483968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075207.840201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075207.840223:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075207.840236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff8b1006025880 x1855007612484096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075207.840246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612484096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075207.840252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075207.840255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075207.840257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484096:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075208.735817:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075209.759961:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075210.783896:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075211.807816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075212.831796:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075212.959770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075212.959787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075212.959794:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075212.959809:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2201 00000100:00080000:1.0:1769075212.959824:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075212.959834:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075212.959886:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612484224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075212.959917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075212.959920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075212.959922:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075212.959926:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2201 00000100:00080000:1.0:1769075212.959929:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075212.959932:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075212.959943:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075212.959953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612484352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075212.959960:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612484224:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075212.959964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075212.959970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075212.959972:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075212.959978:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2201 00000100:00080000:1.0:1769075212.959982:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075212.959987:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075212.959994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075212.960009:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612484480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075212.960020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075212.960023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075212.960026:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075212.960029:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2201 00000100:00080000:1.0:1769075212.960035:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075212.960039:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075212.960043:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075212.960050:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075212.960055:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612484608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075212.960105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff8b1006136d80 x1855007612484224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.960126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612484224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.960136:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.960141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075212.960146:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612484224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075212.960180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075212.960208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075212.960220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff8b1006134700 x1855007612484352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.960230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612484352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.960236:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.960239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075212.960242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075212.960246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075212.960262:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075212.960268:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff8b1006137100 x1855007612484480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.960277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612484480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.960283:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.960285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075212.960287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075212.960291:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075212.960304:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075212.960310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff8b1006135500 x1855007612484608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.960319:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612484608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.960325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.960326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075212.960328:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484608:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075213.855895:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075214.879767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075215.903849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075216.927885:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075217.951946:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075218.079775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.079789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075218.079795:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.079808:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2206 00000100:00080000:1.0:1769075218.079823:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.079832:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075218.079877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612484736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.079914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.079919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.079921:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.079926:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2206 00000100:00080000:1.0:1769075218.079931:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.079935:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075218.079946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.079961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612484864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.079985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.079989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.079991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.079997:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2206 00000100:00080000:1.0:1769075218.080001:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.080006:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075218.080012:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.080026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612484992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075218.080036:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612484736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075218.080042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.080059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075218.080062:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.080068:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2206 00000100:00080000:1.0:1769075218.080072:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.080079:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075218.080086:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.080106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612485120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075218.080161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075218.080210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff8b1006135880 x1855007612484736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.080228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612484736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.080241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.080247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075218.080253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612484736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075218.080264:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075218.080319:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075218.080331:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff8b1006137b80 x1855007612484864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.080341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612484864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.080353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.080356:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075218.080358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075218.080386:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075218.080403:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075218.080410:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff8b1006135c00 x1855007612484992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.080419:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612484992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.080425:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.080427:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075218.080430:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612484992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075218.080433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075218.080445:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075218.080451:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff8b1006136300 x1855007612485120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.080459:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612485120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.080464:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.080466:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075218.080469:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485120:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075218.975857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075219.999879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075221.023815:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075222.047845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075223.071800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075223.199913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.199934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075223.199955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.199979:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2211 00000100:00080000:1.0:1769075223.199995:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.200012:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075223.200175:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612485248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075223.200236:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.200239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075223.200241:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.200278:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2211 00000100:00080000:1.0:1769075223.200282:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.200287:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075223.200298:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075223.200324:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612485376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075223.200335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612485248:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075223.200350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.200354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075223.200356:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.200361:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2211 00000100:00080000:1.0:1769075223.200372:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.200376:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075223.200394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075223.200405:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612485504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075223.200425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.200436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075223.200438:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.200442:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2211 00000100:00080000:1.0:1769075223.200456:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769075223.200456:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075223.200464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075223.200468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075223.200497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612485632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075223.200554:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff8b1006136a00 x1855007612485248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.200582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612485248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.200597:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.200601:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075223.200605:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612485248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075223.200684:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075223.200710:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075223.200721:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff8b1006135180 x1855007612485376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.200731:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612485376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.200737:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.200740:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075223.200742:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075223.200747:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075223.200760:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075223.200768:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff8b1006137800 x1855007612485504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.200776:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612485504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.200782:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.200784:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075223.200786:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075223.200789:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075223.200801:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075223.200808:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff8b1006136300 x1855007612485632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.200816:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612485632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.200832:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.200835:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075223.200837:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485632:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075224.095829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075225.119866:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075226.143857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075227.167834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075228.191822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075228.319761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075228.319774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075228.319778:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075228.319789:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2216 00000100:00080000:1.0:1769075228.319804:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075228.319813:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075228.319867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612485760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075228.319894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075228.319897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075228.319899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075228.319902:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2216 00000100:00080000:1.0:1769075228.319905:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075228.319909:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075228.319918:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075228.319927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612485888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075228.319937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075228.319940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075228.319941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075228.319946:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2216 00000100:00100000:2.0:1769075228.319949:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612485760:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075228.319950:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075228.319956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075228.319961:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075228.319970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612486016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075228.319976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075228.319979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075228.319980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075228.319983:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2216 00000100:00080000:1.0:1769075228.319986:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075228.319988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075228.319991:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075228.319998:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612486144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075228.320038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075228.320075:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff8b1006135c00 x1855007612485760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.320091:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612485760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.320101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.320105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075228.320108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612485760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075228.320155:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075228.320172:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075228.320182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff8b1006137b80 x1855007612485888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.320201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612485888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.320206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.320209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075228.320211:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612485888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075228.320215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075228.320227:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075228.320233:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff8b1006135880 x1855007612486016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.320240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612486016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.320246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.320248:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075228.320249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075228.320254:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075228.320265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075228.320272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff8b1006135500 x1855007612486144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.320280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612486144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.320285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.320287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075228.320289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486144:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075229.215933:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075230.239837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075231.263866:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075232.287938:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075233.311842:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075233.439766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.439774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075233.439778:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.439803:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2221 00000100:00080000:3.0:1769075233.439814:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.439819:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075233.439851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612486272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.439888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.439892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075233.439894:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.439897:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2221 00000100:00080000:3.0:1769075233.439900:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.439902:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075233.439910:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075233.439919:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612486400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.439928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075233.439933:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612486272:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075233.439933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075233.439936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.439943:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2221 00000100:00080000:3.0:1769075233.439946:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.439948:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075233.439951:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075233.439960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612486528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.439967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.439970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075233.439971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.439974:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2221 00000100:00080000:3.0:1769075233.439984:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.439987:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075233.439991:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075233.439998:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612486656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075233.440028:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075233.440068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075233.440087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486656:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075234.335773:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075235.359827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075236.383893:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075236.512286:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075236.512404:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075236.512436:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075236.512513:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075236.512527:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075236.512540:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075236.512722:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075236] req@ffff8b10037df100 x1855007612486400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075236.512767:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612486400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075236.512793:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075236.512800:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075236.512806:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075236.512819:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075236] req@ffff8b10037dd880 x1855007612486528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075236.512834:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612486528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075236.512845:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075236.512850:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075236.512855:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075236.512863:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075236] req@ffff8b10037dea00 x1855007612486656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075236.512880:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612486656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075236.512890:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075236.512896:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075236.512901:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075236.512911:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075236] req@ffff8b10037df800 x1855007612486272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075236.512930:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612486272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075236.512942:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075236.512947:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075236.512952:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612486272:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769075237.407851:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075238.431822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075238.559733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075238.559742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075238.559746:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075238.559755:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2226 00000100:00080000:3.0:1769075238.559765:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075238.559772:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075238.559805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612486784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075238.559833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075238.559837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075238.559838:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075238.559842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2226 00000100:00080000:3.0:1769075238.559844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075238.559847:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075238.559853:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075238.559862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612486912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075238.559870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075238.559875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075238.559876:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075238.559880:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2226 00000100:00080000:3.0:1769075238.559883:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075238.559886:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075238.559889:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075238.559896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612486784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075238.559898:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612487040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075238.559912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075238.559914:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075238.559916:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075238.559919:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2226 00000100:00080000:3.0:1769075238.559922:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075238.559924:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075238.559928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075238.559944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612487168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075238.559994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075238.560032:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff8b10037df480 x1855007612486784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.560057:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612486784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.560066:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.560070:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075238.560073:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612486784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075238.560080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075238.560105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075238.560122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff8b10037dce00 x1855007612486912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.560138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612486912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.560148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.560152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075238.560156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612486912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075238.560186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075238.560213:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075238.560229:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff8b10037dc700 x1855007612487040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.560244:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612487040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.560254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.560258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075238.560261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075238.560268:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075238.560293:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075238.560307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff8b10037df800 x1855007612487168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.560322:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612487168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.560331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.560336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075238.560340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487168:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075239.455857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075240.479962:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075241.503849:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075242.527944:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075243.551832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075243.679723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075243.679735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075243.679740:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075243.679750:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2231 00000100:00080000:3.0:1769075243.679764:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075243.679772:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075243.679818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612487296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075243.679848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075243.679851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075243.679852:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075243.679856:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2231 00000100:00080000:3.0:1769075243.679858:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075243.679876:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075243.679886:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075243.679897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612487424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075243.679906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075243.679910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075243.679911:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075243.679917:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2231 00000100:00100000:1.0:1769075243.679919:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612487296:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075243.679921:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075243.679927:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075243.679932:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075243.679940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612487552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075243.679947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075243.679949:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075243.679951:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075243.679955:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2231 00000100:00080000:3.0:1769075243.679958:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075243.679961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075243.679964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075243.679971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612487680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075243.680001:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075243.680051:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff8b10037dea00 x1855007612487296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.680074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612487296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.680084:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.680089:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075243.680093:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612487296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075243.680134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075243.680165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075243.680182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff8b10037dd880 x1855007612487424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.680194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612487424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.680202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.680206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075243.680209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075243.680215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075243.680235:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075243.680244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff8b10037df100 x1855007612487552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.680260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612487552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.680277:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.680282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075243.680286:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075243.680307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075243.680326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075243.680347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff8b10037ddc00 x1855007612487680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.680360:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612487680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.680366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.680370:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075243.680372:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487680:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075244.575877:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075245.599826:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075246.623890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075247.647831:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075248.671889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075248.799707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075248.799719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075248.799724:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075248.799735:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2236 00000100:00080000:3.0:1769075248.799749:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075248.799757:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075248.799823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dfb80 x1855007612487808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075248.799853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075248.799856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075248.799858:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075248.799862:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2236 00000100:00080000:3.0:1769075248.799865:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075248.799869:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075248.799878:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075248.799889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612487936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075248.799899:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075248.799904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612487808:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075248.799904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075248.799907:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075248.799915:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2236 00000100:00080000:3.0:1769075248.799918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075248.799920:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075248.799923:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075248.799931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612488064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075248.799937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075248.799940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075248.799941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075248.799944:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2236 00000100:00080000:3.0:1769075248.799946:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075248.799948:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075248.799951:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075248.799958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612488192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075248.799997:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075248.800038:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075248] req@ffff8b10037dfb80 x1855007612487808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075248.800056:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dfb80 x1855007612487808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075248.800065:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075248.800069:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075248.800073:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612487808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075248.800096:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487936:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075248.800115:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075248.800125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075248] req@ffff8b10037dc000 x1855007612487936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075248.800134:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612487936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075248.800139:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075248.800142:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075248.800144:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612487936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075248.800148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075248.800160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075248.800166:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075248] req@ffff8b10037ddf80 x1855007612488064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075248.800174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612488064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075248.800180:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075248.800182:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075248.800184:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075248.800187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075248.800198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075248.800204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075248] req@ffff8b10037de300 x1855007612488192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075248.800213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612488192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075248.800218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075248.800220:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075248.800222:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488192:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075249.695896:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075250.719783:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075251.743863:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075252.767823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075253.791843:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075253.919857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075253.919866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075253.919871:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075253.919879:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2242 00000100:00080000:2.0:1769075253.919889:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075253.919895:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075253.919928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612488320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075253.919958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075253.919961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075253.919962:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075253.919965:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2242 00000100:00080000:2.0:1769075253.919967:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075253.919970:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075253.919976:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075253.919985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612488448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075253.919992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075253.919995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075253.919996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075253.919998:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2242 00000100:00080000:2.0:1769075253.920000:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075253.920003:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075253.920005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075253.920014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612488320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075253.920014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612488576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075253.920034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075253.920036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075253.920037:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075253.920040:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2242 00000100:00080000:2.0:1769075253.920042:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075253.920044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075253.920047:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075253.920054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612488704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075253.920089:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075253.920128:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff8b1005d58e00 x1855007612488320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075253.920145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612488320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075253.920153:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075253.920156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075253.920159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612488320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075253.920165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075253.920183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075253.920193:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff8b1005d5bb80 x1855007612488448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075253.920201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612488448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075253.920211:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075253.920214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075253.920216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075253.920237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075253.920250:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075253.920257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff8b1005d59880 x1855007612488576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075253.920265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612488576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075253.920271:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075253.920275:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075253.920278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075253.920285:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075253.920305:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075253.920318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff8b1005d59500 x1855007612488704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075253.920332:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612488704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075253.920339:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075253.920341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075253.920353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488704:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075254.815896:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075255.839795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075256.863836:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075257.887808:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075258.911828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075259.039731:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075259.039742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075259.039747:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075259.039758:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2247 00000100:00080000:3.0:1769075259.039771:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075259.039777:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075259.039817:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612488832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075259.039842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075259.039846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075259.039847:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075259.039851:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2247 00000100:00080000:3.0:1769075259.039853:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075259.039855:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075259.039864:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075259.039873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612488960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075259.039881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075259.039884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075259.039885:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075259.039888:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2247 00000100:00080000:3.0:1769075259.039890:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075259.039894:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075259.039898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075259.039902:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612488832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075259.039905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612489088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075259.039921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075259.039924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075259.039926:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075259.039929:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2247 00000100:00080000:3.0:1769075259.039931:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075259.039933:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075259.039936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075259.039945:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612489216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075259.039984:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075259.040017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075259/real 1769075259] req@ffff8b10037dd180 x1855007612488832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075259.040033:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612488832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075259.040042:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075259.040047:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075259.040050:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612488832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075259.040055:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075259.040071:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075259.040087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075259/real 1769075259] req@ffff8b10037dd500 x1855007612488960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075259.040096:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612488960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075259.040103:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075259.040105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075259.040107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612488960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075259.040129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075259.040142:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075259.040150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075259/real 1769075259] req@ffff8b10037ded80 x1855007612489088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075259.040158:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612489088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075259.040163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075259.040165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075259.040167:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075259.040171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075259.040181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075259.040188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075259/real 1769075259] req@ffff8b10037de680 x1855007612489216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075259.040196:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612489216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075259.040201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075259.040202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075259.040204:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489216:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075259.935755:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075260.959874:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075261.983841:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075263.007920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075264.031917:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075264.159821:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075264.159839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075264.159846:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075264.159863:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2252 00000100:00080000:1.0:1769075264.159879:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075264.159892:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075264.159949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612489344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075264.159992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075264.159997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075264.160000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075264.160005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2252 00000100:00080000:1.0:1769075264.160011:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075264.160017:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075264.160029:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075264.160047:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612489472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075264.160062:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075264.160067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075264.160069:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075264.160075:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2252 00000100:00080000:1.0:1769075264.160079:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075264.160088:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075264.160095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612489344:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769075264.160096:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075264.160125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612489600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075264.160147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075264.160154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075264.160159:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075264.160169:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2252 00000100:00080000:1.0:1769075264.160179:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075264.160204:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769075264.160217:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769075264.160222:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075264.160247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612489728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075264.160296:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff8b1006137100 x1855007612489344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.160336:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612489344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.160368:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.160376:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075264.160385:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612489344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075264.160402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489472:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075264.160454:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075264.160481:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff8b1006134700 x1855007612489472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.160511:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612489472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.160566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.160575:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075264.160583:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075264.160628:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075264.160741:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075264.160769:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff8b1006136d80 x1855007612489600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.160797:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612489600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.160813:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.160820:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075264.160828:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075264.160842:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075264.160904:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075264.160941:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff8b1006136680 x1855007612489728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.160967:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612489728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.160995:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.161002:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075264.161008:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489728:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075265.055947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075266.079976:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075267.103901:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075268.127854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075269.152038:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075269.279868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075269.279883:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075269.279889:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075269.279902:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2257 00000100:00080000:3.0:1769075269.279919:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075269.279927:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075269.279979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612489856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075269.280013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075269.280017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075269.280019:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075269.280023:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2257 00000100:00080000:3.0:1769075269.280026:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075269.280031:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075269.280042:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075269.280055:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612489984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075269.280066:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075269.280070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075269.280073:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075269.280077:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2257 00000100:00080000:3.0:1769075269.280080:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075269.280084:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075269.280088:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075269.280117:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612490112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075269.280128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075269.280133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075269.280135:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075269.280145:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2257 00000100:00100000:2.0:1769075269.280147:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612489856:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075269.280151:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075269.280165:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075269.280172:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075269.280189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612490240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075269.280234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075269.280288:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff8b10037dc380 x1855007612489856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.280307:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612489856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.280317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.280322:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075269.280326:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612489856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075269.280333:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075269.280353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075269.280373:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff8b10037dca80 x1855007612489984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.280384:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612489984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.280391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.280394:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075269.280397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612489984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075269.280421:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075269.280497:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075269.280509:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff8b10037de680 x1855007612490112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.280521:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612490112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.280529:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.280532:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075269.280535:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075269.280544:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075269.280570:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075269.280586:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff8b10037ded80 x1855007612490240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.280606:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612490240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.280634:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.280639:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075269.280643:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490240:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075270.176022:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075271.199836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075272.223926:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075273.247806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075274.271860:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075274.399712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075274.399720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075274.399735:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075274.399743:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2262 00000100:00080000:3.0:1769075274.399753:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075274.399759:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075274.399791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612490368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075274.399822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075274.399825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075274.399826:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075274.399829:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2262 00000100:00080000:3.0:1769075274.399831:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075274.399834:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075274.399848:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075274.399860:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612490496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075274.399861:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612490368:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075274.399876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075274.399879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075274.399880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075274.399883:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2262 00000100:00080000:3.0:1769075274.399886:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075274.399888:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075274.399892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075274.399901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612490624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075274.399909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075274.399911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075274.399913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075274.399916:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2262 00000100:00080000:3.0:1769075274.399918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075274.399920:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075274.399923:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075274.399926:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075274.399934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612490752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075274.399968:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff8b10037dd500 x1855007612490368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075274.399983:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612490368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075274.399991:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075274.399994:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075274.399997:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612490368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075274.400025:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490496:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075274.400040:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075274.400048:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff8b10037dd180 x1855007612490496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075274.400057:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612490496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075274.400073:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075274.400076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075274.400078:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075274.400081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075274.400096:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075274.400102:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff8b10037de300 x1855007612490624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075274.400110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612490624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075274.400119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075274.400122:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075274.400124:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075274.400127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075274.400148:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075274.400154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff8b10037ddf80 x1855007612490752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075274.400161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612490752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075274.400166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075274.400168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075274.400170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612490752:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075275.295893:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075276.319945:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075277.343885:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075278.367824:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075279.391868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075279.519875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.519884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075279.519890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.519902:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2267 00000100:00080000:3.0:1769075279.519919:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.519929:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075279.519973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612490880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075279.520014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.520022:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075279.520026:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.520035:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2267 00000100:00080000:3.0:1769075279.520043:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.520051:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075279.520069:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075279.520103:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dfb80 x1855007612491008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075279.520113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612490880:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075279.520130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.520137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075279.520142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.520152:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2267 00000100:00080000:3.0:1769075279.520160:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.520169:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075279.520178:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075279.520198:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612491136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075279.520209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075279.520303:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff8b10037dc000 x1855007612490880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075279.520342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612490880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075279.520362:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075279.520371:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075279.520378:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612490880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075279.520416:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075279.520460:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075279.520485:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff8b10037dfb80 x1855007612491008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075279.520540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dfb80 x1855007612491008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075279.520557:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075279.520564:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075279.520572:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491008:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769075279.520616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.520624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075279.520629:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.520677:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2267 00000100:00100000:1.0:1769075279.520683:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491136:192.168.204.102@tcp:8:kworker.0 00000100:00080000:3.0:1769075279.520688:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.520699:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075279.520711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075279.520742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612491264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075279.520743:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075279.520784:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff8b10037ddc00 x1855007612491136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075279.520812:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612491136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075279.520829:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075279.520837:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075279.520844:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075279.520868:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075279.520909:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075279.520932:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff8b10037df100 x1855007612491264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075279.520961:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612491264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075279.520983:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075279.520990:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075279.521001:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491264:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075280.415799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075281.439944:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075282.463897:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075283.487852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075284.511850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075284.639747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075284.639758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075284.639776:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075284.639788:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2272 00000100:00080000:3.0:1769075284.639809:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075284.639816:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075284.639865:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612491392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075284.639894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075284.639898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075284.639900:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075284.639903:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2272 00000100:00080000:3.0:1769075284.639905:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075284.639908:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075284.639929:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075284.639938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612491520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075284.639945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075284.639948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075284.639949:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075284.639952:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2272 00000100:00080000:3.0:1769075284.639954:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075284.639956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075284.639960:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075284.639967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612491648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075284.639979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075284.639982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075284.639983:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075284.639988:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2272 00000100:00080000:3.0:1769075284.639991:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075284.639995:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075284.639997:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075284.640004:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612491776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075284.640058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612491392:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769075284.640165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075284.640223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff8b10037dd880 x1855007612491392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075284.640239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612491392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075284.640248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075284.640252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075284.640255:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612491392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075284.640261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075284.640277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075284.640286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff8b10037dea00 x1855007612491520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075284.640294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612491520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075284.640299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075284.640301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075284.640303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075284.640307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075284.640320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075284.640326:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff8b10037df800 x1855007612491648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075284.640348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612491648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075284.640353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075284.640355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075284.640358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075284.640361:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075284.640374:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075284.640380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff8b10037dc700 x1855007612491776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075284.640388:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612491776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075284.640393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075284.640395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075284.640397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612491776:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075285.535875:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075286.559830:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075287.583940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075288.607946:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075289.631912:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075289.759746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075289.759755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075289.759759:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075289.759767:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2277 00000100:00080000:3.0:1769075289.759776:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075289.759782:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075289.759815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612491904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075289.759839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075289.759842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075289.759843:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075289.759846:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2277 00000100:00080000:3.0:1769075289.759848:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075289.759851:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075289.759859:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075289.759867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612492032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075289.759875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075289.759877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075289.759878:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075289.759881:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2277 00000100:00080000:3.0:1769075289.759883:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075289.759886:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075289.759890:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075289.759898:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612492160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075289.759909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075289.759913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075289.759915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075289.759924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2277 00000100:00100000:1.0:1769075289.759926:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612491904:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075289.759930:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075289.759940:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075289.759946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075289.759963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612492288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075289.760017:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075289.760060:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff8b10037dce00 x1855007612491904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075289.760083:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612491904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075289.760131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075289.760135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075289.760139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612491904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075289.760145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492032:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075289.760167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075289.760178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff8b10037df480 x1855007612492032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075289.760187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612492032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075289.760193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075289.760195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075289.760197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075289.760221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075289.760243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075289.760250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff8b10037dc700 x1855007612492160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075289.760258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612492160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075289.760264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075289.760266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075289.760268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075289.760273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075289.760285:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075289.760291:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff8b10037df800 x1855007612492288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075289.760299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612492288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075289.760305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075289.760307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075289.760309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492288:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075290.655953:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075291.680012:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075292.703938:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075293.727888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075294.751931:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075294.879830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075294.879867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075294.879875:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075294.879893:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2282 00000100:00080000:1.0:1769075294.879941:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075294.879954:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075294.880092:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612492416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075294.880138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075294.880145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075294.880148:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075294.880157:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2282 00000100:00080000:1.0:1769075294.880164:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075294.880174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075294.880192:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075294.880209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612492416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075294.880220:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612492544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075294.880241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075294.880245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075294.880248:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075294.880253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2282 00000100:00080000:1.0:1769075294.880258:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075294.880263:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075294.880271:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075294.880286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612492672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075294.880297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075294.880301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075294.880303:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075294.880308:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2282 00000100:00080000:1.0:1769075294.880313:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075294.880316:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075294.880322:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075294.880330:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075294.880341:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612492800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075294.880422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff8b1006134e00 x1855007612492416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075294.880452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612492416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075294.880477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075294.880492:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075294.880498:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612492416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075294.880550:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075294.880580:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075294.880605:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff8b1006134a80 x1855007612492544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075294.880620:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612492544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075294.880647:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075294.880651:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075294.880698:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075294.880728:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075294.880769:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075294.880790:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff8b1006137480 x1855007612492672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075294.880806:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612492672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075294.880816:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075294.880819:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075294.880823:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075294.880831:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075294.880854:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075294.880866:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff8b1006136680 x1855007612492800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075294.880899:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612492800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075294.880909:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075294.880914:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075294.880918:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612492800:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075295.775831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075296.799877:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075297.823772:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075298.847891:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075299.871886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075299.999810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075299.999824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075299.999830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075299.999843:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2288 00000100:00080000:3.0:1769075299.999857:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075299.999865:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075299.999914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612492928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075299.999955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075299.999960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075299.999962:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075299.999967:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2288 00000100:00080000:3.0:1769075299.999972:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075299.999978:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075299.999991:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075300.000001:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612492928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075300.000010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612493056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075300.000022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075300.000025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075300.000027:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075300.000030:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2288 00000100:00080000:3.0:1769075300.000034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075300.000037:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075300.000041:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075300.000051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612493184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075300.000064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075300.000066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075300.000068:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075300.000071:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2288 00000100:00080000:3.0:1769075300.000074:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075300.000077:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075300.000080:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075300.000088:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612493312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075300.000143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075300.000188:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075300.000231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493312:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075300.895856:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075301.919984:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075302.943980:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075303.072060:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769075303.072137:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075303.072166:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075303.072239:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075303.072251:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075303.072265:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769075303.072298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075303] req@ffff8b10037dd880 x1855007612493056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075303.072339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612493056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075303.072354:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075303.072362:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075303.072367:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075303.072377:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075303] req@ffff8b10037df100 x1855007612493184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075303.072394:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612493184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075303.072403:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075303.072407:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075303.072412:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075303.072419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075303] req@ffff8b10037ddc00 x1855007612493312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075303.072431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612493312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075303.072440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075303.072444:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075303.072447:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075303.072456:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075303] req@ffff8b10037dea00 x1855007612492928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075303.072469:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612492928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075303.072477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075303.072482:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075303.072485:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612492928:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769075303.967875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075304.991889:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075305.119772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.119785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075305.119789:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.119803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2293 00000100:00080000:3.0:1769075305.119819:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.119845:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075305.119895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dfb80 x1855007612493440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.119928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.119931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075305.119933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.119936:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2293 00000100:00080000:3.0:1769075305.119940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.119944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075305.119956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075305.119967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612493568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.119976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.119982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769075305.119984:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612493440:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075305.119985:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.119994:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2293 00000100:00080000:3.0:1769075305.119999:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.120004:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075305.120010:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075305.120025:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612493696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.120037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.120043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075305.120046:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.120051:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2293 00000100:00080000:3.0:1769075305.120068:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.120073:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075305.120079:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075305.120086:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075305.120095:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612493824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075305.120139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff8b10037dfb80 x1855007612493440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.120165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dfb80 x1855007612493440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.120179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.120184:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075305.120189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612493440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075305.120223:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075305.120250:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075305.120266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff8b10037dc000 x1855007612493568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.120281:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612493568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.120291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.120295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075305.120300:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075305.120305:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075305.120351:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075305.120365:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff8b10037ddf80 x1855007612493696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.120380:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612493696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.120389:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.120393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075305.120397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075305.120403:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075305.120425:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075305.120439:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff8b10037de300 x1855007612493824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.120452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612493824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.120460:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.120464:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075305.120467:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612493824:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075306.015838:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075307.039931:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075308.063917:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075309.087872:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075310.111917:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075310.239974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075310.239994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075310.240002:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075310.240021:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2298 00000100:00080000:1.0:1769075310.240041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075310.240055:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075310.240137:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612493952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075310.240189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075310.240195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075310.240198:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075310.240204:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2298 00000100:00080000:1.0:1769075310.240211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075310.240217:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075310.240231:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075310.240249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612494080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075310.240264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075310.240269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075310.240274:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075310.240281:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2298 00000100:00080000:1.0:1769075310.240285:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075310.240292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075310.240298:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075310.240311:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612494208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075310.240323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612493952:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075310.240354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075310.240373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075310.240378:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075310.240388:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2298 00000100:00080000:1.0:1769075310.240399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075310.240408:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075310.240420:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075310.240448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612494336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075310.240451:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075310.240519:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff8b1006136d80 x1855007612493952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.240556:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612493952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.240572:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.240578:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075310.240584:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612493952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075310.240593:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075310.240621:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075310.240636:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff8b1006134700 x1855007612494080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.240705:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612494080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.240719:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.240726:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075310.240734:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075310.240817:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075310.240875:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075310.240900:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff8b1006137100 x1855007612494208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.240920:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612494208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.240939:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.240944:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075310.240948:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075310.240956:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075310.240984:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075310.240998:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff8b1006135500 x1855007612494336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.241012:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612494336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.241022:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.241026:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075310.241029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494336:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075311.135948:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075312.159879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075313.183867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075314.207868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075315.231865:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075315.359773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.359789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075315.359796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.359811:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2303 00000100:00080000:3.0:1769075315.359827:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.359838:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075315.359892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612494464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.359936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.359941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075315.359943:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.359947:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2303 00000100:00080000:3.0:1769075315.359951:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.359955:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075315.359967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075315.359985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612494592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075315.359994:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612494464:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075315.360002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.360008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075315.360010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.360016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2303 00000100:00080000:3.0:1769075315.360022:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.360027:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075315.360044:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075315.360063:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612494720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.360082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.360086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075315.360089:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.360094:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2303 00000100:00080000:3.0:1769075315.360098:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.360102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075315.360108:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075315.360120:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075315.360165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075315] req@ffff8b10037dd180 x1855007612494464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075315.360185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612494848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075315.360190:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612494464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075315.360217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075315.360224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075315.360230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612494464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075315.360258:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494592:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075315.360285:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075315.360299:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075315] req@ffff8b10037dd500 x1855007612494592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075315.360315:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612494592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075315.360323:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075315.360328:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075315.360332:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075315.360339:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075315.360364:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075315.360381:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075315] req@ffff8b10037ded80 x1855007612494720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075315.360395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612494720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075315.360405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075315.360409:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075315.360413:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075315.360422:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075315.360443:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075315.360457:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075315] req@ffff8b10037de680 x1855007612494848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075315.360473:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612494848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075315.360483:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075315.360487:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075315.360491:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612494848:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075316.255834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075317.279797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075318.303798:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075319.327865:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075320.351773:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075320.479772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075320.479785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075320.479791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075320.479819:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2308 00000100:00080000:3.0:1769075320.479834:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075320.479843:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075320.479907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612494976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075320.479960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075320.479963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075320.479965:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075320.479969:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2308 00000100:00080000:3.0:1769075320.479972:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075320.479975:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075320.479987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075320.479999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612495104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075320.480007:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612494976:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075320.480011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075320.480016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075320.480017:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075320.480021:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2308 00000100:00080000:3.0:1769075320.480024:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075320.480027:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075320.480030:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075320.480041:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612495232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075320.480049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075320.480052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075320.480053:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075320.480056:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2308 00000100:00080000:3.0:1769075320.480059:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075320.480062:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075320.480065:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075320.480071:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612495360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075320.480108:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075320.480146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff8b10037dca80 x1855007612494976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075320.480164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612494976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075320.480173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075320.480177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075320.480180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612494976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075320.480207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075320.480232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075320.480246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff8b10037dc380 x1855007612495104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075320.480269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612495104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075320.480278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075320.480282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075320.480286:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075320.480293:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075320.480317:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075320.480327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff8b10037de680 x1855007612495232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075320.480340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612495232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075320.480349:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075320.480352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075320.480356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075320.480363:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075320.480402:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075320.480412:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff8b10037ded80 x1855007612495360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075320.480420:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612495360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075320.480431:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075320.480434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075320.480436:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495360:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075321.375914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075322.399824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075323.423780:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075324.447899:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075325.471825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075325.599786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075325.599804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075325.599812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075325.599827:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2313 00000100:00080000:3.0:1769075325.599842:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075325.599854:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075325.599909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612495488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075325.599954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075325.599959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075325.599962:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075325.599967:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2313 00000100:00080000:3.0:1769075325.599972:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075325.599977:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075325.599988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075325.600004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612495616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075325.600018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075325.600022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075325.600025:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075325.600030:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2313 00000100:00080000:3.0:1769075325.600034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075325.600038:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075325.600044:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075325.600058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612495744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075325.600066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612495488:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075325.600102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075325.600109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075325.600114:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075325.600122:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2313 00000100:00080000:3.0:1769075325.600129:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075325.600136:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075325.600146:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075325.600169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612495872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075325.600194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075325.600246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff8b10037dd500 x1855007612495488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075325.600271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612495488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075325.600284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075325.600289:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075325.600294:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612495488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075325.600303:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075325.600333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075325.600359:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff8b10037dd180 x1855007612495616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075325.600374:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612495616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075325.600396:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075325.600400:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075325.600404:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075325.600433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075325.600456:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075325.600469:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff8b10037de300 x1855007612495744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075325.600483:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612495744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075325.600528:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075325.600532:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075325.600537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075325.600545:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075325.600573:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075325.600588:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff8b10037ddf80 x1855007612495872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075325.600602:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612495872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075325.600612:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075325.600616:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075325.600620:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612495872:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075326.495863:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075327.519795:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075328.543894:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075329.567829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075330.591799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075330.719766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.719781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075330.719787:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.719801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2318 00000100:00080000:1.0:1769075330.719815:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.719841:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075330.719881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612496000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.719912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.719915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075330.719917:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.719921:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2318 00000100:00080000:1.0:1769075330.719925:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.719930:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075330.719957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075330.719974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612496128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075330.719981:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612496000:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075330.719998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.720003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075330.720004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.720008:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2318 00000100:00080000:1.0:1769075330.720012:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.720020:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075330.720025:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075330.720054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612496256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.720075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.720078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075330.720081:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.720086:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2318 00000100:00080000:1.0:1769075330.720094:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.720099:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075330.720105:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075330.720107:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075330.720126:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612496384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075330.720171:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff8b1006135880 x1855007612496000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.720193:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612496000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.720205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.720209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075330.720213:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612496000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075330.720251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496128:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075330.720272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075330.720282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff8b1006137b80 x1855007612496128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.720291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612496128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.720297:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.720300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075330.720303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075330.720307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075330.720321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075330.720327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff8b1006135c00 x1855007612496256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.720335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612496256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.720341:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.720342:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075330.720344:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075330.720349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075330.720360:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075330.720372:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff8b1006136300 x1855007612496384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.720386:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612496384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.720392:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.720395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075330.720397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496384:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075331.615844:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075332.639910:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075333.663824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075334.687889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075335.711889:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075335.839833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.839848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075335.839854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.839867:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2323 00000100:00080000:3.0:1769075335.839883:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.839891:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075335.839944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612496512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.840018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.840025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075335.840045:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.840051:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2323 00000100:00080000:3.0:1769075335.840057:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.840065:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075335.840082:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075335.840102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dfb80 x1855007612496640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.840115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.840123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075335.840126:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075335.840127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612496512:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075335.840140:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2323 00000100:00080000:3.0:1769075335.840150:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.840156:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075335.840163:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075335.840201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612496768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.840215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.840220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075335.840222:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.840226:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2323 00000100:00080000:3.0:1769075335.840252:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.840259:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075335.840269:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075335.840283:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075335.840299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612496896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075335.840399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff8b10037dc000 x1855007612496512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.840428:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612496512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.840443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.840449:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075335.840455:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612496512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075335.840488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075335.840526:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075335.840558:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff8b10037dfb80 x1855007612496640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.840582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dfb80 x1855007612496640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.840593:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.840597:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075335.840602:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075335.840610:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075335.840636:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075335.840710:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff8b10037dea00 x1855007612496768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.840726:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612496768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.840737:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.840741:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075335.840745:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075335.840753:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075335.840779:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075335.840791:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff8b10037ddc00 x1855007612496896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.840816:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612496896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.840825:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.840829:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075335.840833:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612496896:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075336.735877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075337.759828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075338.783840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075339.807759:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075340.831799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075340.959759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075340.959768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075340.959772:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075340.959780:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2329 00000100:00080000:2.0:1769075340.959789:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075340.959795:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075340.959857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612497024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075340.959883:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075340.959886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075340.959887:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075340.959890:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2329 00000100:00080000:2.0:1769075340.959893:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075340.959895:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075340.959903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075340.959918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612497152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075340.959927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075340.959929:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612497024:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075340.959930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075340.959934:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075340.959941:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2329 00000100:00080000:2.0:1769075340.959943:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075340.959945:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075340.959950:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075340.959963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612497280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075340.959973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075340.959977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075340.959979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075340.959983:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2329 00000100:00080000:2.0:1769075340.959988:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075340.959993:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769075340.959996:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075340.960004:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075340.960025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612497408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075340.960029:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff8b1005d5ad80 x1855007612497024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075340.960058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612497024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075340.960067:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075340.960071:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075340.960074:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612497024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075340.960092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497152:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075340.960106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075340.960114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff8b1005d5aa00 x1855007612497152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075340.960126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612497152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075340.960134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075340.960137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075340.960141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075340.960148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075340.960171:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075340.960182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff8b1005d5b100 x1855007612497280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075340.960193:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612497280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075340.960201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075340.960204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075340.960207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075340.960213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075340.960268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075340.960280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff8b1005d58380 x1855007612497408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075340.960292:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612497408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075340.960305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075340.960309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075340.960312:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497408:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075341.855827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075342.879820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075343.903857:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075344.927803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075345.951848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075346.079749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075346.079758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075346.079763:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075346.079773:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2334 00000100:00080000:1.0:1769075346.079783:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075346.079811:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075346.079842:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612497536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075346.079863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075346.079866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075346.079868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075346.079871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2334 00000100:00080000:1.0:1769075346.079875:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075346.079878:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075346.079886:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075346.079892:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612497536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075346.079897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612497664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075346.079907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075346.079910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075346.079912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075346.079915:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2334 00000100:00080000:1.0:1769075346.079917:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075346.079920:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075346.079924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075346.079936:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612497792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075346.079946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075346.079952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769075346.079952:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075346.079957:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075346.079964:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2334 00000100:00080000:1.0:1769075346.079970:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075346.079975:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075346.079982:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075346.079989:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff8b1006137800 x1855007612497536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075346.079999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612497920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075346.080009:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612497536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075346.080020:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075346.080025:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075346.080028:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612497536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075346.080047:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497664:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075346.080062:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075346.080071:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff8b1006135180 x1855007612497664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075346.080080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612497664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075346.080086:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075346.080089:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075346.080091:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075346.080095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075346.080107:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075346.080113:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff8b1006136a00 x1855007612497792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075346.080121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612497792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075346.080131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075346.080133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075346.080135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075346.080138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075346.080167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075346.080176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff8b1006136300 x1855007612497920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075346.080184:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612497920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075346.080189:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075346.080191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075346.080193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612497920:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075346.975818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075347.999811:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075349.023777:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075350.047795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075351.071812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075351.199775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075351.199799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075351.199806:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075351.199823:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2339 00000100:00080000:3.0:1769075351.199848:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075351.199858:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075351.199906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612498048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075351.199946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075351.199952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075351.199954:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075351.199960:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2339 00000100:00080000:3.0:1769075351.199965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075351.199970:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075351.199983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075351.199993:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612498048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075351.200001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612498176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075351.200025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075351.200029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075351.200031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769075351.200110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075351.200118:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2339 00000100:00080000:3.0:1769075351.200124:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075351.200130:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075351.200172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff8b10037df100 x1855007612498048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.200193:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612498048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.200203:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.200209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075351.200213:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612498048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075351.200242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498176:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075351.200263:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075351.200274:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff8b10037dd880 x1855007612498176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.200283:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612498176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.200290:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769075351.200291:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1769075351.200293:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075351.200302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498176:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075351.200363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612498304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075351.200384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075351.200388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075351.200389:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075351.200392:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2339 00000100:00080000:3.0:1769075351.200397:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075351.200400:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075351.200406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075351.200413:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075351.200419:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612498432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075351.200439:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075351.200458:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff8b10037df800 x1855007612498304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.200468:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612498304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.200474:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.200476:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075351.200478:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075351.200487:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075351.200501:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075351.200508:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff8b10037dc700 x1855007612498432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.200516:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612498432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.200521:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.200524:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075351.200526:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498432:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075352.095909:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075353.119796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075354.143844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075355.167842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075356.191833:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075356.319734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.319745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075356.319750:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.319761:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2344 00000100:00080000:0.0:1769075356.319772:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.319779:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075356.319816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612498560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.319849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.319853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075356.319856:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.319860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2344 00000100:00080000:0.0:1769075356.319864:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.319868:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075356.319878:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075356.319893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612498688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.319904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.319909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075356.319911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.319915:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2344 00000100:00080000:0.0:1769075356.319920:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.319925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075356.319930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075356.319932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612498560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075356.319945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612498816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.319957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.319961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075356.319963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.319967:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2344 00000100:00080000:0.0:1769075356.320001:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.320007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769075356.320012:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769075356.320015:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075356.320033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612498944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075356.320057:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff8b1006025f80 x1855007612498560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.320080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612498560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.320092:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.320097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075356.320101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612498560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075356.320109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498688:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075356.320135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075356.320148:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff8b1006024a80 x1855007612498688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.320169:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612498688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.320178:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.320182:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075356.320185:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075356.320207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075356.320222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075356.320230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff8b1006026d80 x1855007612498816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.320238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612498816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.320243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.320246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075356.320248:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075356.320251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075356.320261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075356.320267:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff8b1006027b80 x1855007612498944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.320274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612498944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.320288:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.320292:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075356.320296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612498944:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075357.215824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075358.239818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075359.263876:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075360.287817:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075361.311834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075361.439784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.439796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075361.439800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.439812:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2349 00000100:00080000:3.0:1769075361.439826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.439836:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075361.439894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612499072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075361.439930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.439934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075361.439935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.439939:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2349 00000100:00080000:3.0:1769075361.439942:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.439946:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075361.439957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075361.439970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612499200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075361.439979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.439982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075361.439983:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.439987:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2349 00000100:00080000:3.0:1769075361.439990:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.439992:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075361.439996:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075361.440010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612499328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075361.440011:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612499072:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075361.440046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.440050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075361.440052:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.440056:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2349 00000100:00080000:3.0:1769075361.440060:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.440062:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075361.440066:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075361.440076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612499456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075361.440107:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075361.440185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff8b10037df480 x1855007612499072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.440205:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612499072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.440218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.440223:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075361.440227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612499072:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075361.440236:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499200:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075361.440260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075361.440272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff8b10037dce00 x1855007612499200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.440286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612499200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.440295:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.440298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075361.440302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499200:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075361.440328:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075361.440348:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075361.440358:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff8b10037dc700 x1855007612499328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.440371:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612499328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.440389:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.440393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075361.440397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075361.440404:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075361.440422:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075361.440433:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff8b10037df800 x1855007612499456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.440445:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612499456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.440454:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.440457:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075361.440461:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499456:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075362.335845:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075363.359827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075364.383852:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075365.407900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075366.431873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075366.559781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.559793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075366.559819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.559832:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2354 00000100:00080000:3.0:1769075366.559846:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.559853:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075366.559900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612499584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.559927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.559930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075366.559932:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.559935:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2354 00000100:00080000:3.0:1769075366.559943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.559948:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075366.559960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075366.559974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612499712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.559987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.559991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075366.559994:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.559998:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2354 00000100:00080000:3.0:1769075366.560003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.560009:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075366.560016:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075366.560023:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612499584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075366.560034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612499840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.560049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.560052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075366.560054:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.560058:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2354 00000100:00080000:3.0:1769075366.560062:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.560065:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075366.560069:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075366.560078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612499968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075366.560149:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499712:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075366.560185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075366.560202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499968:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075367.455859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075368.479793:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075369.503878:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075369.631986:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075369.632036:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075369.632058:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075369.632102:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075369.632110:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075369.632117:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075369.632183:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075369] req@ffff8b10037ddc00 x1855007612499840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075369.632224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612499840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075369.632238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075369.632244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075369.632249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075369.632257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075369] req@ffff8b10037dea00 x1855007612499968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075369.632269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612499968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075369.632277:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075369.632281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075369.632284:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075369.632292:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075369] req@ffff8b10037dd880 x1855007612499584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075369.632304:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612499584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075369.632318:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075369.632321:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075369.632324:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612499584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075369.632331:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075369] req@ffff8b10037df100 x1855007612499712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075369.632343:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612499712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075369.632351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075369.632354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075369.632357:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612499712:192.168.204.102@tcp:38:kworker.0 00010000:00010000:0.0:1769075370.527836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075371.551864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075371.679825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075371.679833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075371.679837:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075371.679845:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2359 00000100:00080000:0.0:1769075371.679856:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075371.679863:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075371.679900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612500096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075371.679933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075371.679937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075371.679939:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075371.679944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2359 00000100:00080000:0.0:1769075371.679948:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075371.679953:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075371.679963:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075371.679978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612500224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075371.679991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075371.679997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769075371.679998:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612500096:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075371.680000:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075371.680010:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2359 00000100:00080000:0.0:1769075371.680014:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075371.680019:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075371.680025:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075371.680038:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612500352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075371.680049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075371.680052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075371.680055:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075371.680059:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2359 00000100:00080000:0.0:1769075371.680063:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075371.680068:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075371.680073:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075371.680081:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075371.680093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612500480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075371.680125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff8b1006024700 x1855007612500096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.680145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612500096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.680158:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.680162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075371.680166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612500096:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075371.680193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500224:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075371.680220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075371.680234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff8b1006024380 x1855007612500224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.680251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612500224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.680260:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.680264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075371.680267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500224:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075371.680274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075371.680299:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075371.680311:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff8b1006027480 x1855007612500352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.680323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612500352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.680332:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.680336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075371.680340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075371.680355:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075371.680376:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075371.680388:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff8b1006024e00 x1855007612500480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.680410:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612500480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.680419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.680422:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075371.680426:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500480:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075372.575818:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075373.599825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075374.623787:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075375.647861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075376.671993:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075376.799852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075376.799867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075376.799874:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075376.799886:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2364 00000100:00080000:2.0:1769075376.799901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075376.799910:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075376.799962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612500608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075376.800013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075376.800019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075376.800021:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075376.800026:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2364 00000100:00080000:2.0:1769075376.800031:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075376.800035:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075376.800057:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075376.800073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612500736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075376.800088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075376.800093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075376.800096:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075376.800101:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2364 00000100:00080000:2.0:1769075376.800120:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075376.800124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612500608:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075376.800130:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075376.800145:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075376.800162:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612500864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075376.800175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075376.800180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075376.800183:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075376.800188:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2364 00000100:00080000:2.0:1769075376.800193:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075376.800198:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075376.800204:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075376.800232:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612500992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075376.800244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075376.800308:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff8b1005d58380 x1855007612500608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.800335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612500608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.800349:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.800355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075376.800360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612500608:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075376.800371:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500736:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075376.800403:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075376.800419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff8b1005d5b100 x1855007612500736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.800435:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612500736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.800455:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.800459:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075376.800464:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500736:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075376.800519:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075376.800581:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075376.800607:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff8b1005d5aa00 x1855007612500864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.800722:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612500864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.800741:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.800748:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075376.800755:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075376.800780:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075376.800826:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075376.800852:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff8b1005d5ad80 x1855007612500992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.800869:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612500992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.800886:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.800892:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075376.800901:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612500992:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075377.695841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075378.719841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075379.743823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075380.767770:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075381.791880:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075381.919748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075381.919758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075381.919761:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075381.919769:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2370 00000100:00080000:1.0:1769075381.919779:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075381.919784:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075381.919817:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612501120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075381.919846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075381.919849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075381.919850:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075381.919853:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2370 00000100:00080000:1.0:1769075381.919856:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075381.919859:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075381.919867:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075381.919879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612501248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075381.919889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075381.919897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769075381.919899:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612501120:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075381.919901:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075381.919918:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2370 00000100:00080000:1.0:1769075381.919922:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075381.919937:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075381.919944:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075381.919963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612501376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075381.919971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075381.919973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075381.919975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075381.919979:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2370 00000100:00080000:1.0:1769075381.919985:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075381.919988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075381.919992:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075381.919998:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075381.920003:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612501504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075381.920044:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075381] req@ffff8b1006135c00 x1855007612501120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075381.920061:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612501120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075381.920071:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075381.920076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075381.920079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612501120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075381.920104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501248:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075381.920122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075381.920133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075381] req@ffff8b1006137b80 x1855007612501248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075381.920147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612501248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075381.920155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075381.920159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075381.920163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075381.920170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075381.920194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075381.920207:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075381] req@ffff8b1006135880 x1855007612501376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075381.920217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612501376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075381.920223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075381.920225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075381.920227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075381.920232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075381.920247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075381.920254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075381] req@ffff8b1006135500 x1855007612501504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075381.920263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612501504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075381.920269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075381.920271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075381.920273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501504:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075382.815956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075383.839802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075384.863828:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075385.887805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075386.911860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075387.039778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075387.039810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075387.039815:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075387.039835:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2375 00000100:00080000:1.0:1769075387.039859:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075387.039876:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075387.039991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612501632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075387.040051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075387.040107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075387.040111:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075387.040115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612501632:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075387.040129:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2375 00000100:00080000:1.0:1769075387.040134:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075387.040138:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075387.040163:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075387.040188:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612501760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075387.040207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075387.040218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075387.040222:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075387.040226:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2375 00000100:00080000:1.0:1769075387.040228:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075387.040232:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075387.040237:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075387.040258:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612501888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075387.040265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075387.040269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075387.040270:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075387.040290:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2375 00000100:00080000:1.0:1769075387.040294:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075387.040297:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075387.040310:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075387.040337:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612502016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075387.040345:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075387.040384:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075387] req@ffff8b1006137100 x1855007612501632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075387.040408:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612501632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075387.040421:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075387.040430:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075387.040435:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612501632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075387.040480:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501760:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075387.040508:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075387.040523:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075387] req@ffff8b1006134700 x1855007612501760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075387.040540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612501760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075387.040549:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075387.040553:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075387.040557:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501760:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075387.040564:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075387.040588:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075387.040601:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075387] req@ffff8b1006136d80 x1855007612501888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075387.040613:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612501888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075387.040619:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075387.040621:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075387.040624:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612501888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075387.040628:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075387.040677:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075387.040700:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075387/real 1769075387] req@ffff8b1006136680 x1855007612502016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075387.040718:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612502016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075387.040734:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075387.040738:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075387.040742:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502016:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075387.935824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075388.959896:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075389.983804:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075391.007960:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075392.031804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075392.159772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.159783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075392.159788:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.159797:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2380 00000100:00080000:1.0:1769075392.159809:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.159815:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075392.159853:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612502144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075392.159881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.159884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075392.159886:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.159890:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2380 00000100:00080000:1.0:1769075392.159893:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.159896:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075392.159904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075392.159913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612502272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075392.159923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.159926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075392.159928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.159931:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2380 00000100:00080000:1.0:1769075392.159933:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.159938:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075392.159942:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075392.159954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612502400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075392.159955:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612502144:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075392.159974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.159977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075392.159978:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.159981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2380 00000100:00080000:1.0:1769075392.159984:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.159987:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075392.159991:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075392.160001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612502528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075392.160047:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075392.160090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff8b1006137480 x1855007612502144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.160114:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612502144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.160128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.160134:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075392.160140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612502144:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075392.160150:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502272:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075392.160179:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075392.160212:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff8b1006134a80 x1855007612502272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.160230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612502272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.160241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.160246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075392.160251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502272:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075392.160281:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075392.160312:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075392.160329:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff8b1006134e00 x1855007612502400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.160346:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612502400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.160357:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.160361:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075392.160366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075392.160395:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075392.160425:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075392.160442:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff8b1006136680 x1855007612502528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.160458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612502528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.160469:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.160473:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075392.160478:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502528:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075393.055843:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075394.079890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075395.103835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075396.127890:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075397.151817:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075397.279750:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075397.279760:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075397.279763:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075397.279771:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2385 00000100:00080000:1.0:1769075397.279780:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075397.279786:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075397.279818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612502656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075397.279843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075397.279847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075397.279850:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075397.279854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2385 00000100:00080000:1.0:1769075397.279858:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075397.279862:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075397.279872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075397.279885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612502784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075397.279895:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075397.279896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612502656:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075397.279899:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075397.279901:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075397.279905:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2385 00000100:00080000:1.0:1769075397.279909:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075397.279912:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075397.279917:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075397.279926:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612502912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075397.279934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075397.279937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075397.279939:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075397.279943:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2385 00000100:00080000:1.0:1769075397.279955:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075397.279959:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769075397.279959:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769075397.279964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075397.279973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612503040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075397.279993:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff8b1006136d80 x1855007612502656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.280011:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612502656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.280022:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.280027:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075397.280030:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612502656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075397.280051:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502784:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075397.280069:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075397.280078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff8b1006134700 x1855007612502784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.280087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612502784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.280092:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.280094:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075397.280097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075397.280100:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075397.280112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075397.280118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff8b1006137100 x1855007612502912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.280125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612502912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.280135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.280137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075397.280139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612502912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075397.280142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075397.280153:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075397.280159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff8b1006135500 x1855007612503040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.280166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612503040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.280171:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.280172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075397.280174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503040:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075398.175855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075399.199842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075400.223839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075401.247877:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075402.271867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075402.399763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.399773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075402.399785:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.399795:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2390 00000100:00080000:0.0:1769075402.399805:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.399811:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075402.399849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612503168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075402.399873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.399877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075402.399878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.399881:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2390 00000100:00080000:0.0:1769075402.399884:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.399887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075402.399896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075402.399908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612503296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075402.399918:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612503168:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075402.399920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.399927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075402.399929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.399933:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2390 00000100:00080000:0.0:1769075402.399936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.399939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075402.399943:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075402.399954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612503424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075402.399963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.399965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075402.399967:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.399969:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2390 00000100:00080000:0.0:1769075402.399980:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.399983:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769075402.399989:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769075402.399991:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075402.400003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612503552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075402.400028:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff8b1006024e00 x1855007612503168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.400045:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612503168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.400054:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.400058:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075402.400062:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612503168:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075402.400082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503296:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075402.400100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075402.400110:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff8b1006027480 x1855007612503296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.400120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612503296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.400126:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.400129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075402.400132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503296:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075402.400136:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075402.400149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075402.400156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff8b1006024380 x1855007612503424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.400165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612503424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.400171:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.400173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075402.400176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075402.400180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075402.400191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075402.400228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff8b1006024700 x1855007612503552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.400239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612503552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.400246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.400248:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075402.400250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503552:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075403.295775:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075404.319889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075405.343814:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075406.367877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075407.391956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075407.519788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075407.519813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075407.519819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075407.519831:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2395 00000100:00080000:0.0:1769075407.519862:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075407.519869:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075407.519906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612503680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075407.519937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075407.519941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075407.519954:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075407.519959:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2395 00000100:00080000:0.0:1769075407.519962:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075407.519966:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075407.519975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075407.519987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612503808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075407.520007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075407.520011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075407.520014:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075407.520024:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2395 00000100:00100000:3.0:1769075407.520025:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612503680:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075407.520034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075407.520043:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075407.520087:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075407.520178:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612503936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075407.520178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075407.520202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075407.520208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075407.520212:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075407.520218:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2395 00000100:00080000:0.0:1769075407.520226:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075407.520232:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075407.520241:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075407.520252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff8b1006027b80 x1855007612503680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075407.520265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612504064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075407.520323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612503680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.520346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.520352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075407.520359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612503680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075407.520370:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503808:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075407.520407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075407.520425:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff8b1006026d80 x1855007612503808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.520452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612503808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.520480:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.520487:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075407.520495:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075407.520537:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075407.520590:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075407.520607:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff8b1006024a80 x1855007612503936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.520624:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612503936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.520650:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.520696:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075407.520702:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612503936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075407.520712:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075407.520746:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075407.520780:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff8b1006025f80 x1855007612504064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.520798:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612504064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.520809:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.520813:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075407.520818:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504064:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075408.415833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075409.439782:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075410.463896:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075411.487776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075412.511943:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075412.639818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075412.639848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075412.639854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075412.639866:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2400 00000100:00080000:1.0:1769075412.639880:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075412.639889:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075412.639937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612504192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075412.640003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075412.640012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075412.640014:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075412.640019:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2400 00000100:00080000:1.0:1769075412.640023:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075412.640062:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075412.640072:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075412.640084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612504320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075412.640095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075412.640099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075412.640101:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075412.640104:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2400 00000100:00080000:1.0:1769075412.640108:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075412.640131:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075412.640138:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075412.640142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612504192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075412.640154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612504448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075412.640168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075412.640172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075412.640174:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075412.640177:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2400 00000100:00080000:1.0:1769075412.640181:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075412.640185:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075412.640191:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075412.640204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612504576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075412.640228:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075412.640273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff8b1006135880 x1855007612504192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.640295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612504192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.640307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.640312:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075412.640316:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612504192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075412.640325:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504320:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075412.640347:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075412.640380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff8b1006137b80 x1855007612504320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.640396:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612504320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.640406:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.640410:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075412.640414:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075412.640444:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075412.640470:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075412.640493:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff8b1006135c00 x1855007612504448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.640586:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612504448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.640608:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.640616:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075412.640623:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075412.640649:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075412.640749:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075412.640769:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff8b1006136300 x1855007612504576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.640783:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612504576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.640793:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.640796:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075412.640800:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504576:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075413.535858:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075414.559881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075415.583834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075416.607846:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075417.631822:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075417.759790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075417.759804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075417.759810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075417.759822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2405 00000100:00080000:1.0:1769075417.759836:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075417.759844:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075417.759893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612504704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075417.759930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075417.759935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075417.759961:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075417.759969:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2405 00000100:00080000:1.0:1769075417.759984:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075417.759991:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075417.760006:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075417.760028:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612504832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075417.760037:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612504704:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075417.760067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075417.760073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075417.760076:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075417.760081:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2405 00000100:00080000:1.0:1769075417.760086:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075417.760090:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075417.760106:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075417.760122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612504960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075417.760135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075417.760139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075417.760142:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075417.760146:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2405 00000100:00080000:1.0:1769075417.760161:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075417.760172:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769075417.760175:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769075417.760180:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075417.760202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612505088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075417.760242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff8b1006136a00 x1855007612504704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075417.760288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612504704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075417.760310:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075417.760331:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075417.760339:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612504704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075417.760382:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504832:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075417.760445:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075417.760497:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff8b1006135180 x1855007612504832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075417.760529:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612504832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075417.760548:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075417.760556:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075417.760562:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504832:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075417.760583:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075417.760634:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075417.760701:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff8b1006137800 x1855007612504960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075417.760732:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612504960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075417.760743:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075417.760757:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075417.760762:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612504960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075417.760771:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075417.760799:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075417.760812:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff8b1006136300 x1855007612505088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075417.760827:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612505088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075417.760837:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075417.760841:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075417.760845:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505088:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075418.655905:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075419.679885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075420.703927:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075421.727841:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075422.751844:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075422.879917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075422.879925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075422.879929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075422.879939:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2410 00000100:00080000:1.0:1769075422.879969:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075422.879978:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075422.880014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612505216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075422.880054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075422.880059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075422.880062:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075422.880084:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2410 00000100:00080000:1.0:1769075422.880088:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075422.880093:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075422.880105:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075422.880125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612505344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075422.880138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075422.880143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075422.880146:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075422.880154:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2410 00000100:00080000:1.0:1769075422.880158:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075422.880164:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075422.880171:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075422.880184:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612505216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075422.880198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612505472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075422.880228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075422.880232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075422.880234:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075422.880240:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2410 00000100:00080000:1.0:1769075422.880247:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075422.880253:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075422.880259:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075422.880278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612505600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075422.880319:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075422.880376:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff8b1006135c00 x1855007612505216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075422.880398:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612505216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075422.880410:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075422.880416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075422.880421:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612505216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075422.880431:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505344:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075422.880458:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075422.880479:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff8b1006137b80 x1855007612505344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075422.880491:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612505344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075422.880518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075422.880524:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075422.880529:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075422.880563:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075422.880583:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075422.880595:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff8b1006135880 x1855007612505472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075422.880609:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612505472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075422.880617:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075422.880621:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075422.880624:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075422.880629:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075422.880660:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075422.880756:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff8b1006135500 x1855007612505600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075422.880778:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612505600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075422.880790:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075422.880795:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075422.880810:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505600:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075423.775821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075424.799945:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075425.823874:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075426.847986:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075427.871943:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075427.999802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075427.999816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075427.999840:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075427.999851:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2416 00000100:00080000:2.0:1769075427.999864:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075427.999871:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075427.999930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612505728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075427.999976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075427.999983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075427.999986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075427.999991:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2416 00000100:00080000:2.0:1769075427.999996:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075428.000002:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075428.000014:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075428.000033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612505856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075428.000046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612505728:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075428.000052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075428.000060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075428.000063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075428.000068:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2416 00000100:00080000:2.0:1769075428.000072:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075428.000076:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075428.000093:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075428.000109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612505984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075428.000120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075428.000124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075428.000143:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075428.000158:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2416 00000100:00080000:2.0:1769075428.000164:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075428.000169:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075428.000177:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769075428.000184:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075428.000199:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612506112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075428.000275:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075428] req@ffff8b1005d59500 x1855007612505728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075428.000317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612505728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075428.000340:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075428.000349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075428.000375:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612505728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075428.000408:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505856:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075428.000442:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075428.000459:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075428] req@ffff8b1005d59880 x1855007612505856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075428.000474:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612505856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075428.000489:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075428.000494:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075428.000500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505856:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075428.000535:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075428.000581:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075428.000606:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075428] req@ffff8b1005d5bb80 x1855007612505984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075428.000628:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612505984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075428.000639:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075428.000643:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075428.000648:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612505984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075428.000656:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075428.000727:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075428.000744:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075428/real 1769075428] req@ffff8b1005d58e00 x1855007612506112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075428.000759:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612506112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075428.000825:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075428.000830:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075428.000834:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506112:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075428.895929:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075429.919816:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075430.943929:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075431.967957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075432.991855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075433.119781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075433.119792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075433.119797:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075433.119806:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2421 00000100:00080000:0.0:1769075433.119817:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075433.119824:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075433.119859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612506240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075433.119887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075433.119891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075433.119893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075433.119896:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2421 00000100:00080000:0.0:1769075433.119899:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075433.119902:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075433.119911:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075433.119921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612506368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075433.119930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075433.119934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075433.119935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075433.119938:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2421 00000100:00080000:0.0:1769075433.119941:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075433.119944:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075433.119948:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075433.119954:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612506240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075433.119960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612506496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075433.119972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075433.119974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075433.119976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075433.119978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2421 00000100:00080000:0.0:1769075433.119981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075433.119984:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075433.119988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075433.119999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612506624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075433.120078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506368:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075433.120139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.120194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506624:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075434.015977:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075435.039835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075436.063860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075436.191863:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075436.191910:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075436.191924:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075436.191955:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075436.191973:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075436.191978:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075436.192094:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075436] req@ffff8b1006026300 x1855007612506496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075436.192134:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612506496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075436.192149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075436.192156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075436.192161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075436.192171:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075436] req@ffff8b1006024000 x1855007612506624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075436.192184:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612506624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075436.192193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075436.192197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075436.192201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075436.192209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075436] req@ffff8b1006025880 x1855007612506240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075436.192224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612506240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075436.192232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075436.192236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075436.192240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612506240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075436.192248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075436] req@ffff8b1006026a00 x1855007612506368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075436.192261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612506368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075436.192269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075436.192273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075436.192276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506368:192.168.204.102@tcp:38:kworker.0 00010000:00010000:3.0:1769075437.087923:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075438.111752:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075438.239840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.239853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075438.239857:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.239884:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2426 00000100:00080000:2.0:1769075438.239898:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.239906:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075438.239948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612506752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.239993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.239998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075438.240000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.240005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2426 00000100:00080000:2.0:1769075438.240009:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.240015:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075438.240027:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075438.240044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612506880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.240058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.240063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075438.240065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.240086:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2426 00000100:00080000:2.0:1769075438.240090:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.240101:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075438.240104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612506752:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769075438.240110:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075438.240133:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612507008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.240162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.240166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075438.240167:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.240173:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2426 00000100:00080000:2.0:1769075438.240176:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.240179:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075438.240183:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075438.240192:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612507136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075438.240268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075438.240325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff8b1005d5b800 x1855007612506752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.240348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612506752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.240360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.240366:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075438.240371:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612506752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075438.240380:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506880:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075438.240409:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075438.240424:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff8b1005d5b480 x1855007612506880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.240439:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612506880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.240449:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.240453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075438.240459:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612506880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075438.240501:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075438.240690:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075438.240707:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff8b1005d59180 x1855007612507008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.240721:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612507008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.240729:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.240733:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075438.240736:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075438.240744:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075438.240767:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075438.240780:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff8b1005d5a300 x1855007612507136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.240793:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612507136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.240801:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.240805:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075438.240808:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507136:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075439.135851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075440.159811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075441.183815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075442.207812:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075443.231929:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075443.359842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075443.359861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075443.359868:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075443.359884:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2431 00000100:00080000:1.0:1769075443.359901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075443.359912:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075443.359967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612507264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075443.360007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075443.360012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075443.360016:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075443.360022:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2431 00000100:00080000:1.0:1769075443.360028:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075443.360035:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075443.360051:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075443.360070:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612507264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075443.360073:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612507392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075443.360094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075443.360099:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075443.360102:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075443.360108:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2431 00000100:00080000:1.0:1769075443.360113:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075443.360119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075443.360127:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075443.360158:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612507520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075443.360167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075443.360182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075443.360188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075443.360191:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075443.360199:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2431 00000100:00080000:1.0:1769075443.360206:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075443.360213:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075443.360221:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075443.360235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff8b1006137100 x1855007612507264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075443.360242:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612507648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075443.360288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612507264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.360302:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.360307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075443.360311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612507264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075443.360340:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507392:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075443.360366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075443.360379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff8b1006134700 x1855007612507392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.360391:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612507392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.360399:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.360402:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075443.360405:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507392:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075443.360410:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075443.360427:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075443.360435:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff8b1006136d80 x1855007612507520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.360446:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612507520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.360453:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.360456:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075443.360458:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075443.360463:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075443.360477:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075443.360485:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff8b1006136680 x1855007612507648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.360494:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612507648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.360501:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.360504:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075443.360507:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507648:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075444.255853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075445.279800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075446.303819:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075447.327878:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075448.351824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075448.479790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075448.479802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075448.479807:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075448.479818:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2436 00000100:00080000:0.0:1769075448.479830:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075448.479838:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075448.479884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612507776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075448.479914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075448.479918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075448.479920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075448.479940:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2436 00000100:00080000:0.0:1769075448.479943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075448.479946:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075448.479957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075448.479969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612507904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075448.479980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075448.479984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075448.479988:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075448.479989:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612507776:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075448.479994:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2436 00000100:00080000:0.0:1769075448.480000:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075448.480004:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075448.480008:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075448.480034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612508032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075448.480041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075448.480044:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075448.480045:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075448.480048:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2436 00000100:00080000:0.0:1769075448.480050:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075448.480053:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075448.480056:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075448.480064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612508160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075448.480101:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075448.480145:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075448] req@ffff8b1006026680 x1855007612507776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075448.480161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612507776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075448.480170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075448.480174:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075448.480178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612507776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075448.480200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507904:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075448.480219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075448.480228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075448] req@ffff8b1006025180 x1855007612507904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075448.480237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612507904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075448.480243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075448.480245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075448.480247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612507904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075448.480251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075448.480279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075448.480286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075448] req@ffff8b1006027800 x1855007612508032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075448.480295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612508032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075448.480301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075448.480303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075448.480305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075448.480310:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075448.480322:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075448.480329:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075448] req@ffff8b1006027100 x1855007612508160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075448.480338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612508160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075448.480344:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075448.480345:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075448.480348:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508160:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075449.375849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075450.399800:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075451.423816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075452.447884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075453.471869:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075453.599723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075453.599735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075453.599740:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075453.599751:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2441 00000100:00080000:0.0:1769075453.599763:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075453.599770:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075453.599816:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612508288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075453.599841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075453.599845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075453.599846:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075453.599849:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2441 00000100:00080000:0.0:1769075453.599852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075453.599854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075453.599863:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075453.599872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612508416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075453.599879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075453.599894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075453.599896:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075453.599898:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2441 00000100:00080000:0.0:1769075453.599902:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075453.599904:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075453.599907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075453.599914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612508544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075453.599921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075453.599925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075453.599926:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075453.599927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612508288:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075453.599929:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2441 00000100:00080000:0.0:1769075453.599934:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075453.599939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075453.599942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075453.599950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612508672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075453.600036:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075453.600096:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff8b1006027100 x1855007612508288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075453.600113:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612508288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075453.600122:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075453.600127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075453.600130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612508288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075453.600135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508416:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075453.600156:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075453.600178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff8b1006027800 x1855007612508416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075453.600192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612508416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075453.600201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075453.600205:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075453.600209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508416:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075453.600250:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075453.600269:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075453.600277:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff8b1006025180 x1855007612508544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075453.600286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612508544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075453.600292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075453.600295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075453.600297:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075453.600300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075453.600322:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075453.600328:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff8b1006026680 x1855007612508672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075453.600360:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612508672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075453.600366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075453.600368:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075453.600371:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508672:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075454.495828:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075455.519843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075456.543938:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075457.567823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075458.591908:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075458.719926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075458.719941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075458.719949:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075458.719968:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2446 00000100:00080000:1.0:1769075458.719986:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075458.720000:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075458.720049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612508800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075458.720103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075458.720109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075458.720112:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075458.720118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2446 00000100:00080000:1.0:1769075458.720136:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075458.720142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075458.720158:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075458.720184:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612508928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075458.720202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769075458.720212:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612508800:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075458.720212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075458.720220:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075458.720244:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2446 00000100:00080000:1.0:1769075458.720249:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075458.720279:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075458.720288:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075458.720310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612509056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075458.720324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075458.720328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075458.720331:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075458.720336:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2446 00000100:00080000:1.0:1769075458.720344:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075458.720349:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075458.720366:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075458.720381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612509184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075458.720412:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075458.720486:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff8b1006134e00 x1855007612508800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075458.720515:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612508800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.720550:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075458.720556:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075458.720563:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612508800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075458.720623:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508928:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075458.720680:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075458.720735:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff8b1006134a80 x1855007612508928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075458.720749:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612508928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.720758:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075458.720762:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075458.720767:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612508928:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075458.720777:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075458.720806:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075458.720829:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff8b1006137480 x1855007612509056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075458.720856:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612509056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.720873:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075458.720878:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075458.720884:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075458.720897:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075458.720918:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075458.720939:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff8b1006136680 x1855007612509184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075458.720972:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612509184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.720993:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075458.721002:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075458.721013:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509184:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075459.615945:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075460.639953:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075461.663931:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075462.688027:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075463.711940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075463.839809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075463.839826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075463.839834:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075463.839850:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2451 00000100:00080000:1.0:1769075463.839869:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075463.839881:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075463.839945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612509312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075463.839990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075463.839995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075463.839999:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075463.840005:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2451 00000100:00080000:1.0:1769075463.840010:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075463.840016:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075463.840031:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075463.840053:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612509440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075463.840070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075463.840074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075463.840077:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075463.840088:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2451 00000100:00100000:0.0:1769075463.840092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612509312:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075463.840095:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075463.840113:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075463.840125:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075463.840193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612509568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075463.840205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075463.840210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075463.840213:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075463.840219:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2451 00000100:00080000:1.0:1769075463.840243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075463.840250:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769075463.840262:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769075463.840265:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075463.840294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612509696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075463.840353:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff8b1006136d80 x1855007612509312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075463.840392:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612509312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075463.840407:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075463.840413:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075463.840420:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612509312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075463.840464:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509440:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075463.840532:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075463.840568:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff8b1006134700 x1855007612509440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075463.840586:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612509440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075463.840598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075463.840603:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075463.840607:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509440:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075463.840617:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075463.840689:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075463.840706:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff8b1006137100 x1855007612509568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075463.840726:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612509568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075463.840742:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075463.840749:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075463.840755:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075463.840768:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075463.840812:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075463.840836:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff8b1006135500 x1855007612509696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075463.840857:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612509696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075463.840868:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075463.840872:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075463.840877:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509696:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075464.735816:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075465.759947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075466.783883:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075467.807881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075468.831914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075468.959734:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.959746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075468.959751:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.959762:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2457 00000100:00080000:1.0:1769075468.959776:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.959784:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075468.959851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612509824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.959878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.959881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075468.959883:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.959886:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2457 00000100:00080000:1.0:1769075468.959889:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.959891:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075468.959901:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075468.959913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612509952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.959935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.959939:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075468.959941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.959946:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2457 00000100:00080000:1.0:1769075468.959950:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.959956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075468.959971:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075468.959972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612509824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075468.959986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612510080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.959996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.959999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075468.960001:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.960004:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2457 00000100:00080000:1.0:1769075468.960008:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.960011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075468.960016:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075468.960027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612510208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075468.960054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075468.960109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff8b1006135880 x1855007612509824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075468.960138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612509824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075468.960147:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075468.960151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075468.960155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612509824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075468.960161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509952:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075468.960177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075468.960186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff8b1006137b80 x1855007612509952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075468.960194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612509952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075468.960199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075468.960201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075468.960203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612509952:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075468.960225:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075468.960237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075468.960244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff8b1006135c00 x1855007612510080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075468.960253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612510080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075468.960258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075468.960260:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075468.960262:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075468.960266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075468.960276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075468.960283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff8b1006136300 x1855007612510208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075468.960291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612510208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075468.960296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075468.960298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075468.960300:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510208:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075469.855852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075470.879864:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075471.903944:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075472.927849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075473.951895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075474.079797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075474.079811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075474.079830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075474.079844:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2462 00000100:00080000:0.0:1769075474.079876:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075474.079884:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075474.079951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612510336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075474.079992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075474.080014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075474.080016:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075474.080027:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2462 00000100:00080000:0.0:1769075474.080033:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075474.080041:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075474.080071:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075474.080089:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612510464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075474.080096:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612510336:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075474.080106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075474.080117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075474.080119:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075474.080124:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2462 00000100:00080000:0.0:1769075474.080128:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075474.080132:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075474.080137:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075474.080166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612510592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075474.080179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075474.080194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075474.080196:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075474.080201:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2462 00000100:00080000:0.0:1769075474.080206:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075474.080210:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075474.080216:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075474.080229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612510720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075474.080241:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075474.080317:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff8b1006026a00 x1855007612510336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.080338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612510336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.080353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.080358:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075474.080363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612510336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075474.080397:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510464:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075474.080426:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075474.080439:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff8b1006025880 x1855007612510464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.080450:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612510464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.080457:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.080460:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075474.080463:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075474.080468:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.080484:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075474.080491:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff8b1006024000 x1855007612510592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.080500:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612510592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.080506:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.080508:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075474.080510:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075474.080515:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075474.080528:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075474.080540:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff8b1006026300 x1855007612510720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075474.080556:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612510720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075474.080566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075474.080570:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075474.080574:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510720:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075474.975807:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075475.999904:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075477.023916:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075478.047924:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075479.071831:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075479.199807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075479.199815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075479.199819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075479.199826:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2467 00000100:00080000:0.0:1769075479.199837:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075479.199842:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075479.199880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612510848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075479.199904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075479.199907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075479.199925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075479.199928:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2467 00000100:00080000:0.0:1769075479.199931:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075479.199934:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075479.199942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075479.199976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612510976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075479.199985:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612510848:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075479.200004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075479.200010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075479.200013:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075479.200018:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2467 00000100:00080000:0.0:1769075479.200024:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075479.200028:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075479.200036:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075479.200066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612511104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075479.200072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075479.200074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075479.200075:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075479.200078:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2467 00000100:00080000:0.0:1769075479.200081:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075479.200083:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075479.200086:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075479.200093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612511232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075479.200103:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075479.200141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff8b1006025f80 x1855007612510848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075479.200157:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612510848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075479.200165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075479.200168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075479.200171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612510848:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075479.200189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510976:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075479.200207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075479.200239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff8b1006024a80 x1855007612510976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075479.200256:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612510976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075479.200283:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075479.200288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075479.200293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612510976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075479.200302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075479.200329:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075479.200343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff8b1006026d80 x1855007612511104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075479.200358:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612511104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075479.200369:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075479.200373:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075479.200380:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075479.200391:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075479.200428:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075479.200472:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff8b1006027b80 x1855007612511232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075479.200487:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612511232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075479.200495:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075479.200499:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075479.200503:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511232:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075480.095828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075481.119813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075482.143914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075483.167942:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075484.191848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075484.319705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075484.319717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075484.319721:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075484.319733:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2472 00000100:00080000:0.0:1769075484.319745:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075484.319753:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075484.319792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612511360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075484.319820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075484.319851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075484.319854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075484.319861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2472 00000100:00080000:0.0:1769075484.319866:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075484.319873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075484.319882:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075484.319886:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612511360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075484.319896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612511488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075484.319908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075484.319911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075484.319913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075484.319919:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2472 00000100:00080000:0.0:1769075484.319923:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075484.319925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075484.319929:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075484.319941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612511616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075484.319948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075484.319951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075484.319952:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075484.319955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2472 00000100:00080000:0.0:1769075484.319958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075484.319960:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075484.319964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075484.319972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612511744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075484.319991:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075484.320038:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff8b1006024700 x1855007612511360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075484.320055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612511360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075484.320067:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075484.320072:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075484.320077:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612511360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075484.320108:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511488:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075484.320138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075484.320152:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff8b1006024380 x1855007612511488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075484.320164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612511488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075484.320170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075484.320173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075484.320175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075484.320182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075484.320201:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075484.320211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff8b1006027480 x1855007612511616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075484.320224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612511616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075484.320232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075484.320237:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075484.320241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075484.320249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075484.320270:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075484.320279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff8b1006024e00 x1855007612511744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075484.320288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612511744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075484.320293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075484.320296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075484.320298:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612511744:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075485.215800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075486.239821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075487.263887:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075488.287843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075489.311799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075489.439808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075489.439824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075489.439831:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075489.439845:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2477 00000100:00080000:1.0:1769075489.439861:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075489.439878:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075489.439925:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612511872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075489.439952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075489.439955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075489.439958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075489.439962:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2477 00000100:00080000:1.0:1769075489.439965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075489.439969:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075489.439978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075489.439988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612512000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075489.439998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075489.439999:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612511872:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075489.440004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075489.440008:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075489.440016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2477 00000100:00080000:1.0:1769075489.440018:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075489.440021:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075489.440024:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075489.440033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612512128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075489.440039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075489.440042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075489.440043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075489.440046:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2477 00000100:00080000:1.0:1769075489.440048:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075489.440050:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075489.440053:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075489.440064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612512256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075489.440092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075489.440135:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff8b1006137800 x1855007612511872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075489.440159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612511872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075489.440171:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075489.440176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075489.440181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612511872:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075489.440208:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512000:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075489.440229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075489.440239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff8b1006135180 x1855007612512000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075489.440248:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612512000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075489.440254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075489.440256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075489.440258:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075489.440262:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075489.440275:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075489.440281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff8b1006136a00 x1855007612512128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075489.440294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612512128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075489.440299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075489.440301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075489.440303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075489.440307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075489.440326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075489.440333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff8b1006136300 x1855007612512256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075489.440342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612512256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075489.440347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075489.440350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075489.440352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512256:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075490.335838:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075491.359820:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075492.383850:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075493.407849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075494.431845:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075494.559816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075494.559824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075494.559828:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075494.559836:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2482 00000100:00080000:0.0:1769075494.559845:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075494.559851:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075494.559884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612512384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075494.559906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075494.559909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075494.559911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075494.559915:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2482 00000100:00080000:0.0:1769075494.559917:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075494.559920:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075494.559928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075494.559939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612512512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075494.559939:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612512384:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075494.559957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075494.559962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075494.559963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075494.559966:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2482 00000100:00080000:0.0:1769075494.559969:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075494.559972:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075494.559975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075494.559983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612512640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075494.559990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075494.559992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075494.559993:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075494.559995:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2482 00000100:00080000:0.0:1769075494.560006:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075494.560010:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075494.560014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075494.560019:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075494.560025:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612512768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075494.560066:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff8b1006024e00 x1855007612512384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075494.560090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612512384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075494.560103:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075494.560109:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075494.560114:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612512384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075494.560140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512512:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075494.560166:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075494.560179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff8b1006027480 x1855007612512512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075494.560192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612512512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075494.560201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075494.560204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075494.560208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512512:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075494.560215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075494.560240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075494.560254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff8b1006024380 x1855007612512640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075494.560266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612512640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075494.560275:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075494.560278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075494.560281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075494.560287:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075494.560304:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075494.560314:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff8b1006024700 x1855007612512768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075494.560326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612512768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075494.560333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075494.560336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075494.560340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612512768:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075495.455826:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075496.479784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075497.503848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075498.527807:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075499.551798:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075499.679749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075499.679758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075499.679761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075499.679769:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2487 00000100:00080000:0.0:1769075499.679779:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075499.679784:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075499.679817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612512896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075499.679842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075499.679845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075499.679847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075499.679850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2487 00000100:00080000:0.0:1769075499.679852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075499.679854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075499.679861:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075499.679872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612513024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075499.679886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075499.679890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075499.679891:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075499.679895:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2487 00000100:00080000:0.0:1769075499.679897:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075499.679902:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075499.679909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075499.679909:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612512896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075499.679930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612513152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075499.679947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075499.679951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075499.679953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075499.679958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2487 00000100:00080000:0.0:1769075499.679963:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075499.679968:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075499.679975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075499.679987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612513280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075499.680038:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513024:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075499.680076:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075499.680097:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513280:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075500.575809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075501.599820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075502.623822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075502.751920:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075502.751975:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075502.751999:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075502.752049:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075502.752059:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075502.752074:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075502.752153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075502] req@ffff8b1006024a80 x1855007612513152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075502.752191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612513152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075502.752203:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075502.752209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075502.752214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075502.752222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075502] req@ffff8b1006025f80 x1855007612513280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075502.752235:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612513280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075502.752243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075502.752247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075502.752251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075502.752272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075502] req@ffff8b1006027b80 x1855007612512896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075502.752291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612512896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075502.752299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075502.752302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075502.752306:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612512896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075502.752314:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075502] req@ffff8b1006026d80 x1855007612513024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075502.752328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612513024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075502.752336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075502.752340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075502.752343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513024:192.168.204.102@tcp:38:kworker.0 00010000:00010000:2.0:1769075503.647815:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075504.671899:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075504.799836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075504.799852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075504.799871:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075504.799887:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2492 00000100:00080000:1.0:1769075504.799904:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075504.799915:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075504.799978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612513408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075504.800038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075504.800046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075504.800050:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075504.800060:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2492 00000100:00080000:1.0:1769075504.800070:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075504.800081:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075504.800102:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075504.800112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612513408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075504.800141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612513536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075504.800168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075504.800175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075504.800179:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075504.800188:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2492 00000100:00080000:1.0:1769075504.800199:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075504.800215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075504.800218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075504.800236:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075504.800262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612513664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075504.800282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff8b1006135c00 x1855007612513408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075504.800308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075504.800319:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612513408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075504.800322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075504.800331:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075504.800355:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2492 00000100:00080000:1.0:1769075504.800366:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075504.800371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075504.800380:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:3.0:1769075504.800389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075504.800401:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612513408:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769075504.800412:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075504.800446:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075504.800448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612513792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075504.800560:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075504.800586:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff8b1006137b80 x1855007612513536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075504.800610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612513536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075504.800619:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075504.800623:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075504.800627:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075504.800668:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075504.800698:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075504.800715:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff8b1006135880 x1855007612513664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075504.800736:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612513664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075504.800744:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075504.800748:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075504.800751:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075504.800777:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075504.800800:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075504.800813:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff8b1006135500 x1855007612513792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075504.800827:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612513792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075504.800836:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075504.800840:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075504.800843:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612513792:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075505.695923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075506.719775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075507.743877:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075508.767881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075509.792544:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075509.919762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075509.919781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075509.919785:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075509.919795:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2498 00000100:00080000:0.0:1769075509.919808:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075509.919815:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075509.919860:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612513920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075509.919886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075509.919890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075509.919892:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075509.919895:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2498 00000100:00080000:0.0:1769075509.919897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075509.919900:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075509.919909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075509.919918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612514048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075509.919927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075509.919932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075509.919944:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075509.919948:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612513920:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075509.919949:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2498 00000100:00080000:0.0:1769075509.919954:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075509.919959:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075509.919963:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075509.919971:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612514176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075509.919977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075509.919980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075509.919981:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075509.919984:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2498 00000100:00080000:0.0:1769075509.919986:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075509.919989:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075509.919992:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075509.920001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612514304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075509.920040:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075509.920079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff8b1006026300 x1855007612513920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075509.920095:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612513920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075509.920105:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075509.920109:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075509.920112:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612513920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075509.920135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514048:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075509.920151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075509.920159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff8b1006024000 x1855007612514048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075509.920168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612514048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075509.920173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075509.920175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075509.920177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514048:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075509.920181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075509.920202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075509.920208:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff8b1006025880 x1855007612514176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075509.920216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612514176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075509.920222:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075509.920224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075509.920228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075509.920234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075509.920255:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075509.920268:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff8b1006026a00 x1855007612514304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075509.920293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612514304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075509.920302:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075509.920306:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075509.920308:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514304:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075510.815804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075511.839943:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075512.863876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075513.887881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075514.911931:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075515.039757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075515.039770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075515.039775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075515.039798:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2503 00000100:00080000:1.0:1769075515.039811:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075515.039819:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075515.039876:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612514432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075515.039903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075515.039906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075515.039908:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075515.039912:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2503 00000100:00080000:1.0:1769075515.039914:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075515.039918:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075515.039930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075515.039946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612514560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075515.039969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075515.039992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075515.039995:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075515.040003:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2503 00000100:00080000:1.0:1769075515.040011:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075515.040018:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075515.040026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612514432:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769075515.040031:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075515.040068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612514688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075515.040090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075515.040095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075515.040097:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075515.040102:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2503 00000100:00080000:1.0:1769075515.040124:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075515.040130:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075515.040137:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075515.040156:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612514816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075515.040198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075515.040281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075515/real 1769075515] req@ffff8b1006137100 x1855007612514432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075515.040348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612514432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075515.040366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075515.040371:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075515.040375:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612514432:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075515.040382:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514560:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075515.040406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075515.040417:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075515/real 1769075515] req@ffff8b1006134700 x1855007612514560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075515.040427:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612514560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075515.040434:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075515.040437:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075515.040439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075515.040477:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075515.040501:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075515.040516:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075515/real 1769075515] req@ffff8b1006136d80 x1855007612514688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075515.040531:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612514688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075515.040540:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075515.040544:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075515.040548:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075515.040554:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075515.040578:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075515.040593:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075515/real 1769075515] req@ffff8b1006136680 x1855007612514816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075515.040612:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612514816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075515.040622:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075515.040626:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075515.040630:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612514816:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075515.935864:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075516.959874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075517.983762:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075519.007786:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075520.031776:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075520.159801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075520.159831:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075520.159836:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075520.159848:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2508 00000100:00080000:1.0:1769075520.159861:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075520.159869:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075520.159916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612514944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075520.159943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075520.159946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075520.159948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075520.159952:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2508 00000100:00080000:1.0:1769075520.159954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075520.159956:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075520.159967:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075520.159977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612515072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075520.159986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075520.159989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075520.159991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075520.159995:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2508 00000100:00080000:1.0:1769075520.159997:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075520.160001:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075520.160004:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075520.160014:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612515200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075520.160022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075520.160024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075520.160028:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075520.160038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2508 00000100:00080000:1.0:1769075520.160041:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075520.160045:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075520.160049:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075520.160049:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612514944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075520.160059:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612515328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075520.160176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075520.160253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff8b1006137480 x1855007612514944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075520.160287:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612514944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075520.160300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075520.160306:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075520.160311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612514944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075520.160320:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515072:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075520.160358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075520.160379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff8b1006134a80 x1855007612515072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075520.160403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612515072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075520.160419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075520.160425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075520.160431:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515072:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075520.160467:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075520.160495:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075520.160508:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff8b1006134e00 x1855007612515200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075520.160522:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612515200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075520.160531:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075520.160535:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075520.160538:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075520.160544:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075520.160562:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075520.160578:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff8b1006136680 x1855007612515328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075520.160590:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612515328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075520.160598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075520.160601:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075520.160605:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515328:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075521.055911:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075522.079925:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075523.103872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075524.127850:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075525.151894:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075525.279838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075525.279855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075525.279861:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075525.279877:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2513 00000100:00080000:0.0:1769075525.279910:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075525.279923:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075525.279992:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612515456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075525.280041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075525.280048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075525.280052:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075525.280071:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2513 00000100:00080000:0.0:1769075525.280080:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075525.280089:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075525.280107:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075525.280118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612515456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075525.280185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612515584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075525.280197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075525.280201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075525.280203:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075525.280206:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2513 00000100:00080000:0.0:1769075525.280210:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075525.280214:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075525.280219:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075525.280248:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612515712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075525.280260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075525.280274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075525.280277:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075525.280282:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2513 00000100:00080000:0.0:1769075525.280287:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075525.280292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769075525.280314:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769075525.280315:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075525.280328:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612515840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075525.280367:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff8b1006026680 x1855007612515456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075525.280386:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612515456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075525.280396:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075525.280400:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075525.280404:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612515456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075525.280430:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515584:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075525.280449:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075525.280459:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff8b1006025180 x1855007612515584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075525.280469:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612515584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075525.280476:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075525.280479:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075525.280482:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515584:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075525.280489:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075525.280503:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075525.280510:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff8b1006027800 x1855007612515712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075525.280519:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612515712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075525.280533:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075525.280535:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075525.280537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075525.280541:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075525.280554:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075525.280561:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff8b1006027100 x1855007612515840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075525.280570:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612515840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075525.280576:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075525.280578:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075525.280580:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612515840:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075526.175936:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075527.199828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075528.223966:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075529.247925:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075530.271876:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075530.399781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.399791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075530.399796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.399807:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2518 00000100:00080000:0.0:1769075530.399819:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.399825:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075530.399862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612515968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.399896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.399900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075530.399901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.399906:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2518 00000100:00080000:0.0:1769075530.399911:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.399915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075530.399925:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075530.399944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612516096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075530.399957:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612515968:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075530.399960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.399979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075530.399982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.399988:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2518 00000100:00080000:0.0:1769075530.399992:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.399996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075530.400001:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075530.400016:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612516224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.400025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.400029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075530.400031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.400034:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2518 00000100:00080000:0.0:1769075530.400037:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.400040:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075530.400045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075530.400079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612516352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075530.400086:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075530.400164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff8b1006027100 x1855007612515968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075530.400187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612515968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075530.400197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075530.400202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075530.400206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612515968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075530.400251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516096:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075530.400274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075530.400285:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff8b1006027800 x1855007612516096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075530.400299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612516096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075530.400306:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075530.400309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075530.400311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516096:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075530.400315:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075530.400329:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075530.400338:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff8b1006025180 x1855007612516224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075530.400346:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612516224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075530.400352:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075530.400355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075530.400357:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075530.400363:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075530.400389:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075530.400408:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff8b1006026680 x1855007612516352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075530.400424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612516352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075530.400441:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075530.400449:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075530.400455:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516352:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075531.295792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075532.319914:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075533.343876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075534.367950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075535.391880:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075535.519813:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075535.519829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075535.519835:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075535.519850:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2523 00000100:00080000:2.0:1769075535.519866:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075535.519875:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075535.519929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612516480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075535.520022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075535.520026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075535.520028:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075535.520032:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2523 00000100:00080000:2.0:1769075535.520036:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075535.520040:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075535.520051:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075535.520064:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612516608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075535.520075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075535.520079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075535.520081:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075535.520085:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2523 00000100:00080000:2.0:1769075535.520090:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075535.520092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612516480:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075535.520094:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075535.520102:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075535.520111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59c00 x1855007612516736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075535.520118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075535.520121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075535.520123:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075535.520126:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2523 00000100:00080000:2.0:1769075535.520129:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075535.520131:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075535.520135:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075535.520142:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a680 x1855007612516864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075535.520195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075535.520238:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff8b1005d58a80 x1855007612516480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075535.520256:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612516480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075535.520266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075535.520270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075535.520273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612516480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075535.520297:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516608:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075535.520317:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075535.520326:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff8b1005d58000 x1855007612516608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075535.520337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612516608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075535.520343:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075535.520346:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075535.520349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075535.520353:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075535.520367:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075535.520374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff8b1005d59c00 x1855007612516736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075535.520384:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59c00 x1855007612516736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075535.520391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075535.520393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075535.520395:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075535.520399:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075535.520412:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075535.520419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff8b1005d5a680 x1855007612516864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075535.520428:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a680 x1855007612516864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075535.520434:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075535.520436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075535.520439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612516864:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075536.415776:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075537.439892:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075538.463854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075539.487884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075540.511856:0:42052:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1769075540.511905:0:42052:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:2.0:1769075540.511951:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1769075540.511951:0:29100:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769075540.511970:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028480000000 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075540.511971:0:29100:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 2/0,0 mode: PW/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075540.511980:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769075540.512024:0:13998:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1769075540.512025:0:29100:0:(vvp_io.c:1854:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769075540.512153:0:13998:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00000080:00200000:3.0:1769075540.512153:0:29100:0:(vvp_io.c:289:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769075540.512238:0:13998:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869480000020 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075540.512238:0:29100:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 1/0,0 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075540.512261:0:29100:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769075540.512262:0:13998:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769075540.512293:0:29100:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff8b101127b800 lock: ffff8b1005699000/0x68a7113811d806bc lrc: 0/0,0 mode: --/PW res: [0x240000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0x143201ebf3064968 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769075540.512294:0:13998:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff8b101127b800 lock: ffff8b1005699e00/0x68a7113811d8067d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869480000020 nid: local remote: 0x143201ebf3064873 expref: -99 pid: 39691 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075540.512348:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075540.512349:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075540.512352:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075540.512353:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075540.512358:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075540.512358:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075540.512386:0:2366:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8b1005d5a680 x1855007612517120/t0(0) o103->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075540.512386:0:2367:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8b10037dfb80 x1855007612516992/t0(0) o103->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075540.639746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075540.639760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075540.639766:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075540.639778:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2528 00000100:00080000:0.0:1769075540.639813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075540.639825:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075540.639953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612517248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075540.639981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075540.639985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075540.639987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075540.639994:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2528 00000100:00080000:0.0:1769075540.639998:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075540.640005:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075540.640019:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075540.640029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612517248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075540.640039:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612517376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075540.640055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075540.640060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075540.640062:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075540.640067:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2528 00000100:00080000:0.0:1769075540.640072:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075540.640077:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075540.640083:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075540.640096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612517504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075540.640104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075540.640107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075540.640109:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075540.640112:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2528 00000100:00080000:0.0:1769075540.640118:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075540.640121:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075540.640125:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075540.640134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075540.640137:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612517632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075540.640236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff8b1006026a00 x1855007612517248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075540.640271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612517248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075540.640304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075540.640310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075540.640317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612517248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075540.640370:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075540.640402:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075540.640419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff8b1006025880 x1855007612517376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075540.640441:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612517376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075540.640451:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075540.640454:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075540.640458:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075540.640480:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075540.640504:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075540.640518:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff8b1006024000 x1855007612517504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075540.640544:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612517504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075540.640554:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075540.640558:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075540.640561:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075540.640567:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075540.640586:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075540.640596:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff8b1006026300 x1855007612517632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075540.640605:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612517632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075540.640617:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075540.640620:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075540.640622:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517632:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075541.535779:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075542.559956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075543.583897:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075544.607998:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075545.631942:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075545.759880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075545.759912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075545.759920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075545.759935:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2533 00000100:00080000:1.0:1769075545.759953:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075545.759966:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075545.760030:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612517760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075545.760087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075545.760092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075545.760095:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075545.760101:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2533 00000100:00080000:1.0:1769075545.760106:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075545.760111:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075545.760141:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075545.760152:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612517888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075545.760161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075545.760164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075545.760165:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075545.760171:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2533 00000100:00100000:3.0:1769075545.760171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612517760:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075545.760177:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075545.760182:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075545.760185:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075545.760196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612518016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075545.760203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075545.760206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075545.760208:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075545.760210:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2533 00000100:00080000:1.0:1769075545.760213:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075545.760215:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075545.760218:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075545.760227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612518144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075545.760257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075545.760299:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff8b1006136d80 x1855007612517760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075545.760315:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612517760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075545.760325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075545.760329:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075545.760332:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612517760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075545.760355:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075545.760371:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075545.760380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff8b1006134700 x1855007612517888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075545.760389:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612517888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075545.760400:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075545.760403:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075545.760406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612517888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075545.760410:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075545.760421:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075545.760427:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff8b1006137100 x1855007612518016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075545.760435:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612518016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075545.760442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075545.760443:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075545.760445:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075545.760449:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075545.760476:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075545.760483:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff8b1006135500 x1855007612518144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075545.760491:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612518144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075545.760496:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075545.760498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075545.760500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518144:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075546.655916:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075547.680019:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075548.703949:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075549.728057:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075550.751952:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075550.879757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075550.879771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075550.879776:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075550.879787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2538 00000100:00080000:1.0:1769075550.879801:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075550.879809:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075550.879863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612518272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075550.879898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075550.879903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075550.879905:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075550.879909:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2538 00000100:00080000:1.0:1769075550.879914:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075550.879921:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075550.879935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075550.879954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612518400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075550.879967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612518272:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075550.879975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075550.879981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075550.879983:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075550.879987:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2538 00000100:00080000:1.0:1769075550.879991:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075550.879996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075550.880004:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075550.880017:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612518528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075550.880025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075550.880028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075550.880030:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075550.880032:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2538 00000100:00080000:1.0:1769075550.880038:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075550.880041:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075550.880043:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075550.880051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612518656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075550.880102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075550.880143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff8b1006135880 x1855007612518272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075550.880159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612518272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075550.880167:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075550.880171:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075550.880174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612518272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075550.880196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518400:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075550.880215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075550.880223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff8b1006137b80 x1855007612518400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075550.880232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612518400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075550.880238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075550.880240:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075550.880243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075550.880247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075550.880259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075550.880265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff8b1006135c00 x1855007612518528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075550.880273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612518528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075550.880279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075550.880281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075550.880283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075550.880287:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075550.880297:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075550.880303:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff8b1006136300 x1855007612518656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075550.880311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612518656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075550.880316:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075550.880318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075550.880320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518656:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075551.775945:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075552.799831:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075553.823775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075554.847759:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075555.871870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075555.999751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075555.999767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075555.999774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075555.999791:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2544 00000100:00080000:1.0:1769075555.999810:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075555.999821:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075555.999887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612518784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075555.999929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075555.999934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075555.999937:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075555.999943:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2544 00000100:00080000:1.0:1769075555.999947:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075555.999952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075555.999967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075555.999985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612518912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075556.000003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075556.000014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612518784:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075556.000016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075556.000026:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075556.000040:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2544 00000100:00080000:1.0:1769075556.000048:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075556.000058:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075556.000070:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075556.000100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612519040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075556.000123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075556.000130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075556.000133:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075556.000151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075556.000173:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2544 00000100:00080000:1.0:1769075556.000185:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075556.000192:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075556.000202:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075556.000226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612519168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075556.000244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075556] req@ffff8b1006136a00 x1855007612518784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075556.000316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612518784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075556.000333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075556.000339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075556.000344:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612518784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075556.000379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075556.000433:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075556.000460:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075556] req@ffff8b1006135180 x1855007612518912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075556.000477:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612518912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075556.000488:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075556.000493:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075556.000497:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612518912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075556.000505:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075556.000527:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075556.000539:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075556] req@ffff8b1006137800 x1855007612519040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075556.000571:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612519040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075556.000588:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075556.000592:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075556.000595:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075556.000601:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075556.000781:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075556.000803:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075556] req@ffff8b1006136300 x1855007612519168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075556.000820:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612519168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075556.000837:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075556.000844:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075556.000851:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519168:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769075556.896017:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075557.919864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075558.943867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075559.967861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075560.991854:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075561.119755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.119767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075561.119771:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.119782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2549 00000100:00080000:0.0:1769075561.119796:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.119803:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075561.119847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612519296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075561.119871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.119874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075561.119876:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.119879:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2549 00000100:00080000:0.0:1769075561.119882:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.119886:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075561.119896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075561.119915:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612519424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075561.119923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612519296:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075561.119930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.119935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075561.119938:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.119943:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2549 00000100:00080000:0.0:1769075561.119948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.119953:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075561.119959:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075561.119976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612519552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075561.119983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.119986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075561.119988:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.119991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2549 00000100:00080000:0.0:1769075561.119994:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.119997:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075561.120001:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075561.120007:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075561.120011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612519680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075561.120069:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff8b1006026d80 x1855007612519296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.120087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612519296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.120097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.120101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075561.120105:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612519296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075561.120144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075561.120171:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075561.120194:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff8b1006027b80 x1855007612519424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.120211:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612519424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.120220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.120224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075561.120228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075561.120236:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075561.120261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075561.120272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff8b1006025f80 x1855007612519552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.120282:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612519552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.120289:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.120292:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075561.120294:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075561.120298:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075561.120313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075561.120321:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff8b1006024a80 x1855007612519680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.120329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612519680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.120335:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.120337:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075561.120339:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519680:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075562.015869:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075563.039856:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075564.063817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075565.088012:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075566.111855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075566.239789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075566.239802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075566.239809:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075566.239823:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2554 00000100:00080000:0.0:1769075566.239838:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075566.239850:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075566.239897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612519808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075566.239945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075566.239951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075566.239953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075566.239958:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2554 00000100:00080000:0.0:1769075566.239963:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075566.239969:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075566.240010:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075566.240029:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612519936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075566.240044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075566.240044:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612519808:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075566.240058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075566.240063:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075566.240069:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2554 00000100:00080000:0.0:1769075566.240084:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075566.240089:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075566.240097:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075566.240116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612520064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075566.240126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075566.240129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075566.240132:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075566.240137:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2554 00000100:00080000:0.0:1769075566.240155:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075566.240161:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075566.240182:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075566.240198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612520192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075566.240206:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075566.240277:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075566.240323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520192:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075567.135807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075568.159867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075569.183792:0:13998:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075569.311856:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769075569.311891:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075569.311907:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075569.311936:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075569.311942:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769075569.311947:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075569.311998:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075569] req@ffff8b1006024e00 x1855007612520192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075569.312024:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612520192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075569.312036:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075569.312041:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075569.312045:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075569.312053:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075569] req@ffff8b1006024380 x1855007612519936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075569.312065:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612519936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075569.312072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075569.312076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075569.312079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612519936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075569.312086:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075569] req@ffff8b1006027480 x1855007612520064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075569.312097:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612520064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075569.312104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075569.312108:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075569.312111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075569.312117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075569] req@ffff8b1006024700 x1855007612519808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075569.312129:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612519808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075569.312136:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075569.312139:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075569.312143:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612519808:192.168.204.102@tcp:250:kworker.0 00010000:00010000:3.0:1769075570.207931:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075571.231840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075571.359790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075571.359821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075571.359826:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075571.359839:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2559 00000100:00080000:0.0:1769075571.359853:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075571.359862:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075571.359904:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612520320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075571.359930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075571.359934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075571.359935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075571.359940:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2559 00000100:00080000:0.0:1769075571.359943:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075571.359946:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075571.359957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075571.359969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612520448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075571.359984:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612520320:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075571.359986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075571.359996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075571.359998:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075571.360002:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2559 00000100:00080000:0.0:1769075571.360005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075571.360008:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075571.360012:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075571.360024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612520576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075571.360040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075571.360045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075571.360047:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075571.360052:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2559 00000100:00080000:0.0:1769075571.360065:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075571.360071:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769075571.360074:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769075571.360083:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075571.360119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612520704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075571.360138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff8b1006024700 x1855007612520320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075571.360160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612520320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075571.360175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075571.360180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075571.360186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612520320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075571.360236:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075571.360268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075571.360284:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff8b1006027480 x1855007612520448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075571.360301:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612520448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075571.360311:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075571.360316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075571.360320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075571.360329:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075571.360395:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075571.360411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff8b1006024380 x1855007612520576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075571.360426:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612520576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075571.360436:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075571.360439:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075571.360443:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075571.360449:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075571.360472:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075571.360485:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff8b1006024e00 x1855007612520704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075571.360500:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612520704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075571.360509:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075571.360513:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075571.360517:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612520704:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075572.255841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075573.279823:0:42052:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1769075573.279920:0:13998:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769075573.279939:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b10029ac200/0x68a7113811d8073a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x143201ebf306504c expref: -99 pid: 41554 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769075573.279959:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075573.279995:0:13998:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8b101127b800 lock: ffff8b1005699400/0x68a7113811d806ca lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x143201ebf306503e expref: -99 pid: 39885 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769075573.280073:0:13998:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff8b1005d58700 x1855007612443392/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 576/608 e 0 to 0 dl 1769074988 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769075573.280088:0:13998:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff8b101127b800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769075573.280113:0:13998:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff8b1005d59c00 x1855007612520960/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:2.0:1769075573.280142:0:13998:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8b1005d59c00 x1855007612520960/t0(0) o35->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769075573.280155:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075574.303746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075574.303831:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075575.327745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075575.327841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075576.351818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075576.351924:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075576.479884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075576.479923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075576.479932:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075576.479952:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2564 00000100:00080000:1.0:1769075576.479990:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075576.480002:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075576.480068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612521088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075576.480115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075576.480119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075576.480122:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075576.480127:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2564 00000100:00080000:1.0:1769075576.480132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075576.480137:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075576.480150:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075576.480165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612521216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075576.480177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075576.480182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075576.480183:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075576.480187:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2564 00000100:00080000:1.0:1769075576.480190:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075576.480194:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075576.480199:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075576.480209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612521088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075576.480215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612521344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075576.480229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075576.480233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075576.480234:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075576.480239:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2564 00000100:00080000:1.0:1769075576.480242:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075576.480246:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075576.480251:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075576.480327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612521472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075576.480385:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075576.480449:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff8b1006135c00 x1855007612521088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075576.480475:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612521088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075576.480490:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075576.480497:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075576.480502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612521088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075576.480513:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521216:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075576.480544:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075576.480559:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff8b1006137b80 x1855007612521216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075576.480576:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612521216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075576.480600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075576.480604:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075576.480626:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075576.480726:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075576.480761:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075576.480778:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff8b1006135880 x1855007612521344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075576.480795:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612521344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075576.480829:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075576.480834:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075576.480838:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075576.480846:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075576.480879:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075576.480902:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff8b1006135500 x1855007612521472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075576.480928:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612521472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075576.480945:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075576.480952:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075576.480959:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075577.375771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075577.375900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075578.399780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075578.399944:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075579.423806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075579.423818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075580.447758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075580.447888:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075581.471781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075581.471884:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075581.599813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075581.599825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075581.599830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075581.599840:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2569 00000100:00080000:0.0:1769075581.599853:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075581.599861:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075581.599903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612521600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075581.599938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075581.599943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075581.599944:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075581.599949:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2569 00000100:00080000:0.0:1769075581.599952:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075581.599956:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075581.599965:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075581.599979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612521728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075581.599991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075581.599994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075581.599997:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075581.600001:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2569 00000100:00080000:0.0:1769075581.600005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075581.600009:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075581.600014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075581.600028:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612521856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075581.600041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612521600:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075581.600046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075581.600065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075581.600069:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075581.600075:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2569 00000100:00080000:0.0:1769075581.600078:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075581.600082:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075581.600089:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075581.600112:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612521984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075581.600187:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075581.600280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075581] req@ffff8b1006024a80 x1855007612521600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075581.600309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612521600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075581.600335:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075581.600341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075581.600348:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612521600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075581.600360:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521728:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075581.600394:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075581.600410:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075581] req@ffff8b1006025f80 x1855007612521728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075581.600427:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612521728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075581.600437:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075581.600442:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075581.600446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521728:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075581.600482:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075581.600506:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075581.600519:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075581] req@ffff8b1006027b80 x1855007612521856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075581.600534:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612521856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075581.600544:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075581.600548:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075581.600552:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075581.600561:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075581.600581:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075581.600593:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075581] req@ffff8b1006026d80 x1855007612521984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075581.600608:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612521984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075581.600618:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075581.600622:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075581.600625:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612521984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075582.495703:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075582.495826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075583.519793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075583.519836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075584.543696:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075584.543785:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075585.567753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075585.567797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075586.591831:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075586.591841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075586.719786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075586.719797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075586.719802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075586.719813:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2574 00000100:00080000:2.0:1769075586.719826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075586.719833:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075586.719880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612522112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075586.719907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075586.719911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075586.719913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075586.719916:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2574 00000100:00080000:2.0:1769075586.719919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075586.719921:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075586.719930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075586.719939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612522240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075586.719946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075586.719949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075586.719950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075586.719953:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2574 00000100:00080000:2.0:1769075586.719955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075586.719957:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075586.719961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075586.719968:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612522368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075586.719974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075586.719976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075586.719978:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075586.719981:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2574 00000100:00080000:2.0:1769075586.719984:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075586.719989:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075586.719989:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612522112:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769075586.719995:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075586.720003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612522496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075586.720067:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075586.720123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff8b1005d58000 x1855007612522112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075586.720140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612522112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075586.720149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075586.720154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075586.720157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612522112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075586.720163:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522240:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075586.720195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075586.720209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff8b1005d58a80 x1855007612522240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075586.720224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612522240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075586.720233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075586.720236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075586.720238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522240:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075586.720244:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075586.720261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075586.720269:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff8b1005d5a300 x1855007612522368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075586.720277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612522368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075586.720283:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075586.720285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075586.720287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075586.720311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075586.720322:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075586.720329:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff8b1005d59180 x1855007612522496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075586.720342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612522496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075586.720347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075586.720349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075586.720350:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075587.615741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075587.615784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075588.639758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075588.639817:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075589.663786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075589.663901:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075590.687768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075590.687870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075591.711719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075591.711823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075591.839693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075591.839705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075591.839709:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075591.839720:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2579 00000100:00080000:2.0:1769075591.839732:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075591.839740:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075591.839785:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612522624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075591.839815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075591.839819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075591.839821:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075591.839826:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2579 00000100:00080000:2.0:1769075591.839830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075591.839836:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075591.839848:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075591.839863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612522752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075591.839873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075591.839877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075591.839879:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075591.839882:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2579 00000100:00080000:2.0:1769075591.839885:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075591.839888:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075591.839892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075591.839903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612522880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075591.839910:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612522624:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075591.839916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075591.839925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075591.839938:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075591.839943:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2579 00000100:00080000:2.0:1769075591.839963:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075591.839967:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075591.839972:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075591.839989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612523008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075591.840089:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075591.840124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff8b1005d5b480 x1855007612522624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075591.840140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612522624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075591.840148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075591.840152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075591.840154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612522624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075591.840160:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522752:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075591.840175:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075591.840183:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff8b1005d5b800 x1855007612522752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075591.840223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612522752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075591.840232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075591.840235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075591.840239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075591.840289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075591.840318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075591.840346:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff8b1005d58e00 x1855007612522880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075591.840363:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612522880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075591.840384:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075591.840388:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075591.840393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612522880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075591.840400:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075591.840428:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075591.840443:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff8b1005d5bb80 x1855007612523008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075591.840458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612523008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075591.840475:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075591.840495:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075591.840499:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075592.735765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075592.735830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075593.759817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075593.759842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075594.783718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075594.783804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075595.807712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075595.807759:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075596.831718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075596.831766:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075596.959720:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075596.959731:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075596.959736:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075596.959747:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2585 00000100:00080000:2.0:1769075596.959759:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075596.959765:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075596.959798:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612523136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075596.959824:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075596.959827:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075596.959828:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075596.959831:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2585 00000100:00080000:2.0:1769075596.959834:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075596.959837:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075596.959843:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075596.959852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612523264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075596.959858:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075596.959861:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075596.959862:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075596.959864:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2585 00000100:00080000:2.0:1769075596.959866:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075596.959868:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075596.959871:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075596.959878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612523392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075596.959889:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075596.959892:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075596.959893:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075596.959896:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2585 00000100:00080000:2.0:1769075596.959898:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075596.959901:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612523136:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075596.959903:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075596.959911:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075596.959920:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612523520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075596.959979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075596.960010:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff8b1005d59880 x1855007612523136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075596.960024:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612523136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.960032:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075596.960035:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075596.960038:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612523136:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075596.960043:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523264:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075596.960057:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075596.960065:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff8b1005d59500 x1855007612523264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075596.960073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612523264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.960083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075596.960085:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075596.960087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523264:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075596.960091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075596.960102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075596.960107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff8b1005d5ad80 x1855007612523392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075596.960115:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612523392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.960120:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075596.960122:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075596.960124:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075596.960143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075596.960154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075596.960160:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff8b1005d5aa00 x1855007612523520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075596.960167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612523520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.960172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075596.960173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075596.960176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075597.855861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075597.855960:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075598.879763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075598.879876:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075599.903859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075599.903939:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075600.927795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075600.927812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075601.951748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075601.951945:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075602.079883:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.079900:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075602.079908:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.079929:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2590 00000100:00080000:0.0:1769075602.079950:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.079963:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075602.080031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612523648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.080070:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.080077:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075602.080081:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.080093:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2590 00000100:00080000:0.0:1769075602.080103:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.080111:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075602.080130:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075602.080156:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612523776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075602.080168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612523648:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075602.080179:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.080191:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075602.080194:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.080200:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2590 00000100:00080000:0.0:1769075602.080208:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.080218:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075602.080226:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075602.080247:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612523904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.080269:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.080275:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075602.080280:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.080285:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2590 00000100:00080000:0.0:1769075602.080290:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.080310:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769075602.080311:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769075602.080328:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075602.080345:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612524032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075602.080402:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff8b1006026300 x1855007612523648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.080447:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612523648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.080471:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.080481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075602.080490:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612523648:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075602.080535:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523776:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075602.080587:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075602.080614:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff8b1006024000 x1855007612523776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.080696:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612523776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.080722:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.080734:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075602.080742:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075602.080762:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075602.080820:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075602.080881:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff8b1006025880 x1855007612523904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.080922:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612523904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.080940:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.080949:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075602.080958:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612523904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075602.080973:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075602.081015:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075602.081037:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff8b1006026a00 x1855007612524032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.081053:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612524032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.081064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.081068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075602.081072:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075602.975801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075602.975922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075603.999730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075603.999771:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075605.023721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075605.023960:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075606.047832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075606.047965:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075607.071727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075607.071823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075607.199756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075607.199809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075607.199814:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075607.199824:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2595 00000100:00080000:2.0:1769075607.199835:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075607.199842:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075607.199881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612524160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075607.199909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075607.199912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075607.199913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075607.199916:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2595 00000100:00080000:2.0:1769075607.199919:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075607.199922:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075607.199929:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075607.199938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612524288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075607.199945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075607.199948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075607.199950:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075607.199952:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2595 00000100:00080000:2.0:1769075607.199954:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075607.199958:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612524160:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075607.199959:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075607.199967:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075607.199979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612524416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075607.199986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075607.199988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075607.199990:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075607.199993:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2595 00000100:00080000:2.0:1769075607.199996:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075607.199999:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075607.200002:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075607.200009:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612524544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075607.200019:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075607.200051:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff8b1005d5b100 x1855007612524160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.200065:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612524160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.200073:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.200076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075607.200079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612524160:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075607.200102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524288:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075607.200121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075607.200131:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff8b1005d58380 x1855007612524288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.200141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612524288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.200148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.200150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075607.200153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524288:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075607.200189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075607.200203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075607.200221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff8b1005d5aa00 x1855007612524416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.200233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612524416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.200247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.200250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075607.200252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075607.200256:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075607.200268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075607.200275:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff8b1005d5ad80 x1855007612524544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.200283:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612524544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.200288:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.200290:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075607.200292:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524544:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769075608.095992:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075608.096011:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075609.119722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075609.119846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075610.143797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075610.143857:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075611.167749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075611.167804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075612.191771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075612.191933:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075612.319802:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075612.319826:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075612.319831:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075612.319842:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2600 00000100:00080000:0.0:1769075612.319855:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075612.319863:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075612.319906:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612524672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075612.319943:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075612.319947:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075612.319949:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075612.319956:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2600 00000100:00080000:0.0:1769075612.319960:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075612.319964:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075612.319974:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075612.319990:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612524800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075612.320004:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075612.320008:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075612.320010:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075612.320018:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612524672:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075612.320020:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2600 00000100:00080000:0.0:1769075612.320029:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075612.320035:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075612.320041:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075612.320055:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612524928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075612.320066:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075612.320070:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075612.320072:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075612.320077:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2600 00000100:00080000:0.0:1769075612.320092:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075612.320096:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075612.320101:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075612.320105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075612.320119:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612525056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075612.320184:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff8b1006026680 x1855007612524672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.320228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612524672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.320258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.320266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075612.320273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612524672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075612.320303:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524800:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075612.320335:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075612.320350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff8b1006025180 x1855007612524800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.320365:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612524800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.320374:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.320378:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075612.320382:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524800:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075612.320389:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075612.320408:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075612.320418:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff8b1006027800 x1855007612524928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.320431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612524928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.320440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.320442:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075612.320446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612524928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075612.320452:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075612.320469:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075612.320479:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff8b1006027100 x1855007612525056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.320491:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612525056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.320500:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.320503:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075612.320506:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075613.215739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075613.215901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075614.239758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075614.239822:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075615.263815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075615.263977:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075616.287774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075616.287873:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075617.311750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075617.311789:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075617.439794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075617.439807:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075617.439812:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075617.439826:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2605 00000100:00080000:1.0:1769075617.439840:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075617.439848:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075617.439896:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612525184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075617.439923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075617.439926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075617.439928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075617.439931:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2605 00000100:00080000:1.0:1769075617.439933:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075617.439937:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075617.439946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075617.439957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612525312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075617.439964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075617.439968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075617.439969:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075617.439972:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2605 00000100:00080000:1.0:1769075617.439975:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075617.439981:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075617.439981:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612525184:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769075617.439988:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075617.439996:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612525440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075617.440004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075617.440007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075617.440008:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075617.440011:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2605 00000100:00080000:1.0:1769075617.440013:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075617.440015:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075617.440018:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075617.440028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612525568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075617.440065:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075617.440105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff8b1006137100 x1855007612525184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075617.440120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612525184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075617.440128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075617.440132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075617.440136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612525184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075617.440141:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525312:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075617.440156:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075617.440164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff8b1006134700 x1855007612525312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075617.440173:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612525312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075617.440179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075617.440181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075617.440183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525312:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075617.440205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075617.440218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075617.440224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff8b1006136d80 x1855007612525440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075617.440233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612525440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075617.440238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075617.440240:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075617.440242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075617.440245:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075617.440256:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075617.440262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff8b1006136680 x1855007612525568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075617.440281:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612525568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075617.440298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075617.440301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075617.440303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075618.335739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075618.335803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075619.359717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075619.359921:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075620.383756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075620.383902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075621.407737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075621.407789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075622.431809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075622.431825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075622.559742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075622.559761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075622.559766:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075622.559777:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2610 00000100:00080000:1.0:1769075622.559790:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075622.559798:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075622.559843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612525696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075622.559870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075622.559873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075622.559875:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075622.559878:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2610 00000100:00080000:1.0:1769075622.559880:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075622.559883:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075622.559893:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075622.559905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612525824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075622.559914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075622.559917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075622.559919:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075622.559926:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2610 00000100:00080000:1.0:1769075622.559930:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075622.559930:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612525696:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075622.559941:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075622.559946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075622.559960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612525952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075622.559968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075622.559971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075622.559973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075622.559976:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2610 00000100:00080000:1.0:1769075622.559978:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075622.559981:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075622.559984:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075622.559992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612526080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075622.560010:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075622.560060:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff8b1006134e00 x1855007612525696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075622.560075:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612525696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075622.560084:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075622.560088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075622.560091:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612525696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075622.560113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525824:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075622.560130:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075622.560139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff8b1006134a80 x1855007612525824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075622.560147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612525824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075622.560157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075622.560160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075622.560162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525824:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075622.560166:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075622.560178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075622.560185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff8b1006137480 x1855007612525952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075622.560192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612525952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075622.560197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075622.560199:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075622.560201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612525952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075622.560205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075622.560216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075622.560222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff8b1006136680 x1855007612526080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075622.560229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612526080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075622.560234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075622.560236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075622.560239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075623.455719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075623.455808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075624.479742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075624.479812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075625.503766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075625.503825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075626.527755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075626.527810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075627.551774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075627.551838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075627.679741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.679753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075627.679757:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.679769:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2615 00000100:00080000:3.0:1769075627.679782:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.679790:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075627.679835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612526208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.679869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.679872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075627.679874:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.679878:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2615 00000100:00080000:3.0:1769075627.679881:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.679884:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075627.679895:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075627.679907:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612526208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075627.679914:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612526336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.679929:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.679934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075627.679936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.679941:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2615 00000100:00080000:3.0:1769075627.679944:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.679947:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075627.679952:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075627.679963:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612526464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.679970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.679973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075627.679974:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.679976:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2615 00000100:00080000:3.0:1769075627.679978:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.679980:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075627.679983:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075627.679986:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075627.679995:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612526592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075627.680031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff8b10037dc000 x1855007612526208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.680047:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612526208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.680057:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.680060:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075627.680063:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612526208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075627.680086:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526336:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075627.680102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075627.680111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff8b10037ddf80 x1855007612526336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.680121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612526336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.680127:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.680129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075627.680132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526336:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075627.680135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075627.680148:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075627.680154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff8b10037de300 x1855007612526464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.680162:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612526464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.680167:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.680169:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075627.680171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075627.680183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075627.680199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075627.680206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff8b10037dd180 x1855007612526592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.680214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612526592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.680225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.680227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075627.680229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075628.575702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075628.575783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075629.599848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075629.599848:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075630.623717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075630.623798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075631.647767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075631.647854:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075632.671757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075632.671809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075632.799711:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075632.799720:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075632.799725:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075632.799733:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2620 00000100:00080000:0.0:1769075632.799744:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075632.799750:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075632.799785:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612526720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075632.799807:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075632.799810:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075632.799811:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075632.799814:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2620 00000100:00080000:0.0:1769075632.799817:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075632.799819:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075632.799826:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075632.799835:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612526848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075632.799843:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075632.799846:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075632.799848:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075632.799850:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2620 00000100:00080000:0.0:1769075632.799853:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075632.799855:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075632.799859:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075632.799865:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612526720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075632.799868:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612526976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075632.799878:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075632.799880:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075632.799881:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075632.799884:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2620 00000100:00080000:0.0:1769075632.799886:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075632.799888:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075632.799891:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075632.799898:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612527104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075632.799949:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075632.799975:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075632.799987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075633.695754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075633.695833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075634.719793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075634.719809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075635.743766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075635.743810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075635.871866:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075635.871904:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075635.871920:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075635.871950:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075635.871956:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075635.871965:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075635.872019:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075635] req@ffff8b1006025180 x1855007612526976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075635.872046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612526976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075635.872059:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075635.872065:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075635.872069:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075635.872076:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075635] req@ffff8b1006026680 x1855007612527104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075635.872091:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612527104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075635.872099:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075635.872103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075635.872107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075635.872114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075635] req@ffff8b1006027100 x1855007612526720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075635.872127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612526720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075635.872135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075635.872138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075635.872142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612526720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075635.872150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075635] req@ffff8b1006027800 x1855007612526848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075635.872165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612526848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075635.872172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075635.872176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075635.872179:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612526848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075636.767760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075636.767860:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075637.791726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075637.791811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075637.919786:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075637.919793:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075637.919798:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075637.919806:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2626 00000100:00080000:2.0:1769075637.919817:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075637.919823:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075637.919852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612527232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075637.919885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075637.919890:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075637.919893:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075637.919899:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2626 00000100:00080000:2.0:1769075637.919905:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075637.919910:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075637.919918:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612527232:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769075637.919922:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075637.919940:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612527360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075637.919954:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075637.919961:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075637.919963:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075637.919968:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2626 00000100:00080000:2.0:1769075637.919973:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075637.919978:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769075637.919984:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769075637.919988:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075637.920003:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612527488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075637.920011:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075637.920014:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075637.920016:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075637.920021:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2626 00000100:00080000:2.0:1769075637.920026:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075637.920029:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075637.920035:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075637.920046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff8b1005d59500 x1855007612527232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.920051:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612527616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075637.920086:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612527232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075637.920099:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075637.920104:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075637.920109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612527232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075637.920138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527360:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075637.920165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075637.920180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff8b1005d59880 x1855007612527360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075637.920196:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612527360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075637.920205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075637.920209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075637.920214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527360:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075637.920221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075637.920286:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075637.920300:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff8b1005d5bb80 x1855007612527488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075637.920314:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612527488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075637.920338:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075637.920341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075637.920345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075637.920351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075637.920377:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075637.920390:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff8b1005d58e00 x1855007612527616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075637.920402:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612527616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075637.920410:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075637.920414:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075637.920418:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075638.815732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075638.815821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075639.839805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075639.839846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075640.863724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075640.863779:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075641.887795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075641.887799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075642.911725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075642.911827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075643.039882:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075643.039896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075643.039903:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075643.039919:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2631 00000100:00080000:3.0:1769075643.039957:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075643.039970:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075643.040043:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612527744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075643.040093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075643.040101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075643.040105:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075643.040113:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2631 00000100:00080000:3.0:1769075643.040119:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075643.040125:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075643.040137:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075643.040213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612527744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075643.040230:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612527872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075643.040255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075643.040260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075643.040263:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075643.040271:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2631 00000100:00080000:3.0:1769075643.040280:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075643.040287:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075643.040296:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075643.040314:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612528000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075643.040322:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075643.040343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075643.040348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075643.040351:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075643.040358:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2631 00000100:00080000:3.0:1769075643.040364:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075643.040370:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075643.040378:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075643.040391:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff8b10037dd500 x1855007612527744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075643.040395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612528128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075643.040508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612527744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075643.040531:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075643.040538:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075643.040543:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612527744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075643.040571:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527872:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075643.040590:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075643.040599:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff8b10037ded80 x1855007612527872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075643.040608:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612527872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075643.040614:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075643.040617:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075643.040620:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612527872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075643.040624:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075643.040636:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075643.040642:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff8b10037de680 x1855007612528000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075643.040651:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612528000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075643.040697:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075643.040700:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075643.040702:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075643.040707:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075643.040723:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075643.040732:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075643/real 1769075643] req@ffff8b10037dc380 x1855007612528128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075643.040740:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612528128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075643.040746:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075643.040749:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075643.040751:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528128:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075643.935784:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075643.935814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075644.959731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075644.959871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075645.983749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075645.983795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075647.007726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075647.007796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075648.031716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075648.031797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075648.159723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.159735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075648.159740:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.159751:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2636 00000100:00080000:3.0:1769075648.159764:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.159772:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075648.159816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612528256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.159843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.159846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075648.159848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.159851:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2636 00000100:00080000:3.0:1769075648.159854:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.159857:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075648.159866:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075648.159877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612528384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.159884:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.159887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075648.159888:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.159890:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2636 00000100:00080000:3.0:1769075648.159892:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.159895:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075648.159898:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075648.159906:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612528512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.159912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.159915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075648.159916:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.159919:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2636 00000100:00080000:3.0:1769075648.159923:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075648.159929:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612528256:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075648.159929:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075648.159936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075648.159944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612528640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075648.160008:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075648.160045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075648] req@ffff8b10037dca80 x1855007612528256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075648.160062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612528256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075648.160071:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075648.160088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075648.160092:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612528256:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075648.160098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528384:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075648.160114:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075648.160124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075648] req@ffff8b10037dc380 x1855007612528384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075648.160133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612528384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075648.160139:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075648.160141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075648.160144:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075648.160147:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075648.160159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075648.160165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075648] req@ffff8b10037de680 x1855007612528512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075648.160173:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612528512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075648.160187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075648.160189:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075648.160191:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075648.160219:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075648.160232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075648.160239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075648] req@ffff8b10037ded80 x1855007612528640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075648.160247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612528640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075648.160258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075648.160261:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075648.160263:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075649.055739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075649.055799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075650.079710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075650.079805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075651.103778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075651.103791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075652.127716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075652.127916:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075653.151811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075653.151916:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075653.279820:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075653.279840:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075653.279849:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075653.279866:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2641 00000100:00080000:2.0:1769075653.279883:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075653.279893:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075653.279985:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612528768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075653.280062:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075653.280067:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075653.280070:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075653.280075:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2641 00000100:00080000:2.0:1769075653.280080:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075653.280084:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075653.280115:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075653.280138:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612528896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075653.280154:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075653.280158:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075653.280161:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075653.280168:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2641 00000100:00080000:2.0:1769075653.280177:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075653.280180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612528768:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075653.280186:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075653.280205:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075653.280235:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612529024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075653.280250:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075653.280255:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075653.280258:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075653.280262:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2641 00000800:00000100:0.0:1769075653.280270:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769075653.280271:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075653.280290:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075653.280298:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075653.280323:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612529152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075653.280325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff8b1005d5b800 x1855007612528768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075653.280361:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612528768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075653.280385:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075653.280392:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075653.280397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612528768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075653.280417:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528896:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075653.280446:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075653.280457:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff8b1005d5b480 x1855007612528896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075653.280466:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612528896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075653.280472:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075653.280474:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075653.280476:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612528896:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075653.280498:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075653.280511:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075653.280518:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff8b1005d59180 x1855007612529024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075653.280526:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612529024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075653.280531:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075653.280533:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075653.280535:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075653.280540:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075653.280550:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075653.280557:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff8b1005d5a300 x1855007612529152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075653.280565:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612529152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075653.280570:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075653.280572:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075653.280573:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075654.175690:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075654.175745:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075655.199704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075655.199822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075656.223731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075656.223774:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075657.247753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075657.247846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075658.271787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075658.271838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075658.399796:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.399807:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075658.399812:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.399841:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2646 00000100:00080000:0.0:1769075658.399857:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.399868:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075658.399954:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612529280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.399989:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.399993:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075658.399995:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.399998:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2646 00000100:00080000:0.0:1769075658.400006:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.400011:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075658.400021:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075658.400033:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612529408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.400043:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075658.400064:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612529280:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075658.400066:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075658.400072:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.400088:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2646 00000100:00080000:0.0:1769075658.400092:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.400097:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075658.400103:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075658.400121:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612529536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.400143:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.400147:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075658.400149:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.400153:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2646 00000100:00080000:0.0:1769075658.400158:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075658.400164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075658.400171:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075658.400178:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075658.400192:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612529664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075658.400204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff8b1006026a00 x1855007612529280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.400228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612529280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.400248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.400254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075658.400257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612529280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075658.400279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529408:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075658.400299:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075658.400312:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff8b1006025880 x1855007612529408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.400326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612529408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.400336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.400348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075658.400352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075658.400358:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075658.400376:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075658.400388:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff8b1006024000 x1855007612529536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.400401:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612529536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.400426:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.400431:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075658.400434:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075658.400441:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075658.400479:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075658.400492:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff8b1006026300 x1855007612529664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.400505:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612529664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.400514:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.400517:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075658.400520:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075659.295765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075659.295847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075660.319768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075660.319792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075661.343720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075661.343809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075662.367748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075662.367811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075663.391733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075663.391799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075663.519732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075663.519744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075663.519749:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075663.519760:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2651 00000100:00080000:1.0:1769075663.519773:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075663.519781:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075663.519850:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612529792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075663.519877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075663.519881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075663.519883:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075663.519886:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2651 00000100:00080000:1.0:1769075663.519890:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075663.519894:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075663.519904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075663.519914:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612529792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075663.519916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612529920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075663.519930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075663.519932:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075663.519933:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075663.519937:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2651 00000100:00080000:1.0:1769075663.519940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075663.519942:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075663.519945:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075663.519955:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612530048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075663.519961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075663.519964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075663.519966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075663.519969:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2651 00000100:00080000:1.0:1769075663.519991:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075663.519996:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769075663.520002:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769075663.520005:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075663.520022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612530176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075663.520048:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff8b1006136d80 x1855007612529792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075663.520080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612529792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075663.520093:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075663.520099:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075663.520104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612529792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075663.520140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529920:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075663.520160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075663.520171:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff8b1006134700 x1855007612529920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075663.520182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612529920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075663.520188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075663.520191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075663.520193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612529920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075663.520197:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075663.520216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075663.520223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff8b1006137100 x1855007612530048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075663.520232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612530048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075663.520238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075663.520240:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075663.520243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075663.520247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075663.520258:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075663.520264:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff8b1006135500 x1855007612530176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075663.520272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612530176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075663.520283:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075663.520285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075663.520287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075664.415718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075664.415822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075665.439822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075665.439828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075666.463751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075666.463813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075667.487701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075667.487811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075668.511779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075668.511861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075668.639767:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075668.639776:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075668.639780:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075668.639788:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2656 00000100:00080000:2.0:1769075668.639798:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075668.639803:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075668.639837:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612530304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075668.639862:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075668.639865:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075668.639866:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075668.639869:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2656 00000100:00080000:2.0:1769075668.639871:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075668.639874:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075668.639881:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075668.639889:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612530432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075668.639897:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075668.639900:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075668.639901:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075668.639904:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2656 00000100:00080000:2.0:1769075668.639906:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075668.639908:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075668.639911:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075668.639920:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612530560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075668.639927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612530304:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075668.639929:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075668.639938:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075668.639940:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075668.639944:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2656 00000100:00080000:2.0:1769075668.639945:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075668.639947:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075668.639950:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075668.639960:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612530688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075668.640003:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075668.640040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff8b1005d58a80 x1855007612530304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075668.640061:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612530304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075668.640072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075668.640077:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075668.640082:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612530304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075668.640089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530432:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075668.640125:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075668.640139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff8b1005d58000 x1855007612530432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075668.640152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612530432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075668.640160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075668.640164:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075668.640168:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075668.640192:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075668.640214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075668.640226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff8b1005d5a300 x1855007612530560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075668.640239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612530560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075668.640247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075668.640250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075668.640254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075668.640260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075668.640277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075668.640287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff8b1005d59180 x1855007612530688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075668.640299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612530688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075668.640307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075668.640310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075668.640313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075669.535726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075669.535827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075670.559758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075670.559825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075671.583751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075671.583827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075672.607857:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075672.607868:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075673.631755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075673.631797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075673.759777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.759786:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075673.759790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.759802:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2661 00000100:00080000:3.0:1769075673.759814:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.759821:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075673.759856:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612530816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075673.759883:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.759888:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075673.759890:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.759896:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2661 00000100:00080000:3.0:1769075673.759901:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.759905:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075673.759916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075673.759919:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612530816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075673.759935:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612530944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075673.759949:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.759954:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075673.759956:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.759961:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2661 00000100:00080000:3.0:1769075673.759965:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.759969:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075673.759975:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075673.759977:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075673.759990:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612531072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075673.760009:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff8b10037dd500 x1855007612530816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075673.760009:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.760019:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075673.760022:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.760030:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2661 00000100:00100000:1.0:1769075673.760033:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612530816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075673.760036:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.760045:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:1.0:1769075673.760047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.760053:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075673.760056:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612530816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075673.760073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530944:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075673.760089:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075673.760097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff8b10037dd180 x1855007612530944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.760105:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612530944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.760112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.760114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075673.760117:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612530944:192.168.204.102@tcp:38:kworker.0 00000008:00100000:3.0:1769075673.760117:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075673.760127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075673.760138:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612531200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075673.760144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075673.760155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff8b10037de300 x1855007612531072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.760165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612531072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.760171:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.760174:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075673.760176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075673.760187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075673.760199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075673.760205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff8b10037ddf80 x1855007612531200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.760212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612531200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.760219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.760221:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075673.760223:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075674.655769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075674.655890:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075675.679714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075675.679782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075676.703777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075676.703847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075677.727772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075677.727838:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075678.751721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075678.751810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075678.879716:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075678.879728:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075678.879733:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075678.879745:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2666 00000100:00080000:3.0:1769075678.879758:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075678.879766:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075678.879809:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612531328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075678.879840:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075678.879843:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075678.879844:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075678.879848:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2666 00000100:00080000:3.0:1769075678.879851:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075678.879855:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075678.879865:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075678.879875:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612531456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075678.879884:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075678.879887:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075678.879889:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075678.879893:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612531328:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075678.879895:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2666 00000100:00080000:3.0:1769075678.879900:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075678.879904:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075678.879908:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075678.879917:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612531584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075678.879924:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075678.879927:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075678.879929:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075678.879933:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2666 00000100:00080000:3.0:1769075678.879937:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075678.879941:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075678.879947:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075678.879960:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612531712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075678.879982:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075678.880030:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff8b10037dc000 x1855007612531328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.880067:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612531328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.880077:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.880081:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075678.880084:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612531328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075678.880107:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075678.880127:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075678.880138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff8b10037df100 x1855007612531456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.880146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612531456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.880153:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.880155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075678.880158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075678.880161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075678.880174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075678.880181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff8b10037dd880 x1855007612531584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.880203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612531584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.880218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.880222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075678.880226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075678.880233:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075678.880256:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075678.880265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff8b10037dea00 x1855007612531712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.880273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612531712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.880279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.880281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075678.880283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075679.775790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075679.775817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075680.799705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075680.799802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075681.823729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075681.823791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075682.847705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075682.847806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075683.871756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075683.871856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075683.999796:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075683.999811:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075683.999816:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075683.999830:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2672 00000100:00080000:2.0:1769075683.999844:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075683.999854:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075683.999913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612531840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075683.999945:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075683.999948:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075683.999950:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075683.999954:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2672 00000100:00080000:2.0:1769075683.999957:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075683.999961:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075683.999972:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075683.999987:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612531968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075683.999999:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075684.000004:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075684.000007:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075684.000012:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2672 00000100:00080000:2.0:1769075684.000018:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075684.000023:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075684.000030:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075684.000035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612531840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075684.000043:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612532096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075684.000054:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075684.000057:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075684.000058:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075684.000062:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2672 00000100:00080000:2.0:1769075684.000065:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075684.000068:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075684.000072:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075684.000083:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612532224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075684.000126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075684.000181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075684] req@ffff8b1005d5b480 x1855007612531840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075684.000206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612531840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075684.000217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075684.000221:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075684.000225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612531840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075684.000233:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531968:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075684.000253:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075684.000263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075684] req@ffff8b1005d5b800 x1855007612531968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075684.000272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612531968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075684.000278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075684.000281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075684.000283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612531968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075684.000309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075684.000321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075684.000327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075684] req@ffff8b1005d58e00 x1855007612532096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075684.000335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612532096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075684.000341:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075684.000343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075684.000344:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075684.000348:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075684.000359:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075684.000366:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075684] req@ffff8b1005d5bb80 x1855007612532224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075684.000375:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612532224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075684.000380:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075684.000382:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075684.000384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075684.895746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075684.895802:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075685.919757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075685.919827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075686.943859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075686.943859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075687.967734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075687.967797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075688.991714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075688.991775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075689.119772:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.119787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075689.119793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.119806:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2677 00000100:00080000:3.0:1769075689.119821:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.119830:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075689.119883:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612532352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.119918:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.119929:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.119931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.119936:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2677 00000100:00080000:3.0:1769075689.119940:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.119945:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075689.119959:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075689.119971:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612532352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075689.119977:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612532480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.119991:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.119995:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075689.119997:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.120002:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2677 00000100:00080000:3.0:1769075689.120006:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.120011:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075689.120018:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.120031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612532608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075689.120041:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075689.120045:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075689.120047:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075689.120054:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2677 00000800:00000100:1.0:1769075689.120055:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075689.120061:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075689.120068:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075689.120074:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075689.120085:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612532736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075689.120113:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff8b10037ddc00 x1855007612532352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075689.120138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612532352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075689.120148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075689.120152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075689.120155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612532352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075689.120180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075689.120207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075689.120218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff8b10037df800 x1855007612532480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075689.120227:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612532480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075689.120238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075689.120241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075689.120243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075689.120247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075689.120260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075689.120266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff8b10037dc700 x1855007612532608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075689.120274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612532608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075689.120280:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075689.120281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075689.120283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075689.120287:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075689.120298:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075689.120304:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff8b10037dce00 x1855007612532736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075689.120312:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612532736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075689.120317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075689.120319:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075689.120321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075690.015765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075690.015821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075691.039728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075691.039778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075692.063748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075692.063875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075693.087771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075693.087873:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075694.111882:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075694.111933:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075694.239730:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075694.239742:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075694.239746:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075694.239757:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2682 00000100:00080000:3.0:1769075694.239770:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075694.239776:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075694.239829:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612532864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075694.239857:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075694.239860:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075694.239861:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075694.239865:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2682 00000100:00080000:3.0:1769075694.239868:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075694.239870:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075694.239879:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075694.239891:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612532992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075694.239899:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075694.239901:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075694.239902:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075694.239905:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2682 00000100:00080000:3.0:1769075694.239907:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075694.239912:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075694.239914:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612532864:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769075694.239916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075694.239932:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612533120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075694.239941:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075694.239943:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075694.239945:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075694.239948:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2682 00000100:00080000:3.0:1769075694.239951:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075694.239954:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075694.239959:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075694.239967:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612533248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075694.239993:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075694.240030:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff8b10037df480 x1855007612532864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075694.240046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612532864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075694.240055:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075694.240059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075694.240062:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612532864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075694.240068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075694.240083:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075694.240092:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff8b10037dce00 x1855007612532992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075694.240108:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612532992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075694.240114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075694.240116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075694.240118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612532992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075694.240141:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075694.240154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075694.240162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff8b10037dc700 x1855007612533120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075694.240170:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612533120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075694.240176:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075694.240178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075694.240180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075694.240183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075694.240194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075694.240199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff8b10037df800 x1855007612533248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075694.240240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612533248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075694.240252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075694.240257:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075694.240261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075695.135776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075695.135911:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075696.159756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075696.160224:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075697.183712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075697.183986:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075698.207718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075698.208006:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075699.231725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075699.231785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075699.359749:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075699.359761:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075699.359766:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075699.359777:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2687 00000100:00080000:3.0:1769075699.359790:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075699.359796:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075699.359852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612533376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075699.359888:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075699.359892:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075699.359893:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075699.359897:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2687 00000100:00080000:3.0:1769075699.359900:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075699.359904:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075699.359915:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075699.359928:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612533504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075699.359937:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075699.359941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612533376:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075699.359942:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075699.359946:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075699.359954:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2687 00000100:00080000:3.0:1769075699.359956:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075699.359959:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075699.359963:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075699.359972:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612533632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075699.359995:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075699.359999:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075699.360001:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075699.360005:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2687 00000100:00080000:3.0:1769075699.360009:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075699.360012:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075699.360017:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075699.360027:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612533760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075699.360129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075699.360183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075699.360201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075700.255736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075700.255842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075701.279804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075701.279841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075702.303736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075702.303880:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075702.432080:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769075702.432152:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075702.432202:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075702.432266:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075702.432298:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075702.432334:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075702.432354:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075702] req@ffff8b10037dea00 x1855007612533504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075702.432398:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612533504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075702.432415:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075702.432424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075702.432429:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075702.432440:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075702] req@ffff8b10037dd880 x1855007612533632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075702.432456:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612533632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075702.432465:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075702.432470:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075702.432474:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075702.432482:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075702] req@ffff8b10037df100 x1855007612533760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075702.432497:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612533760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075702.432507:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075702.432512:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075702.432516:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612533760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075702.432525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075702] req@ffff8b10037ddc00 x1855007612533376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075702.432542:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612533376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075702.432551:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075702.432557:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075702.433131:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612533376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075703.327742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075703.327917:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075704.351844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075704.351982:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075704.479793:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.479803:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075704.479807:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.479815:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2692 00000100:00080000:3.0:1769075704.479826:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.479832:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075704.479862:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612533888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075704.479891:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.479894:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075704.479895:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.479899:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2692 00000100:00080000:3.0:1769075704.479901:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.479904:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075704.479910:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075704.479918:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612534016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075704.479926:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.479929:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075704.479931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.479934:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2692 00000100:00080000:3.0:1769075704.479938:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.479941:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075704.479944:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075704.479951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612534144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075704.479959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612533888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075704.479960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.479967:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075704.479970:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.479974:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2692 00000100:00080000:3.0:1769075704.479976:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.479978:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075704.479981:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075704.479991:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612534272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075704.480034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075704.480069:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff8b10037ddc00 x1855007612533888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075704.480088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612533888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075704.480098:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075704.480103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075704.480107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612533888:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075704.480115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534016:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075704.480154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075704.480168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff8b10037df100 x1855007612534016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075704.480182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612534016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075704.480191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075704.480195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075704.480198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075704.480225:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075704.480247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075704.480259:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff8b10037dd880 x1855007612534144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075704.480271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612534144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075704.480279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075704.480283:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075704.480287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075704.480293:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075704.480313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075704.480325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff8b10037dea00 x1855007612534272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075704.480336:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612534272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075704.480344:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075704.480348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075704.480351:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075705.375789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075705.375889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075706.399791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075706.399869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075707.423766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075707.423837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075708.447870:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075708.447873:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075709.471747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075709.471815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075709.599736:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.599755:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075709.599764:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.599781:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2697 00000100:00080000:3.0:1769075709.599800:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.599811:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075709.599879:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612534400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075709.599919:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.599925:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075709.599928:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.599935:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2697 00000100:00080000:3.0:1769075709.599940:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.599947:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075709.599963:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075709.599982:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612534528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075709.599991:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612534400:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075709.599997:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.600001:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075709.600004:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.600009:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2697 00000100:00080000:3.0:1769075709.600014:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.600019:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075709.600025:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075709.600037:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612534656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075709.600074:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.600079:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075709.600081:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.600086:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2697 00000100:00080000:3.0:1769075709.600091:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.600096:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075709.600103:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075709.600116:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612534784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075709.600132:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075709.600202:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff8b10037dc000 x1855007612534400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075709.600230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612534400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075709.600246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075709.600254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075709.600261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612534400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075709.600298:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534528:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075709.600333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075709.600352:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff8b10037ddf80 x1855007612534528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075709.600370:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612534528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075709.600382:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075709.600389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075709.600394:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075709.600404:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075709.600438:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075709.600455:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff8b10037de300 x1855007612534656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075709.600485:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612534656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075709.600500:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075709.600506:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075709.600514:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075709.600528:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075709.600565:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075709.600583:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff8b10037dd180 x1855007612534784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075709.600600:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612534784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075709.600612:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075709.600617:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075709.600621:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612534784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075710.495768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075710.495868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075711.519743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075711.519809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075712.543781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075712.543961:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075713.567835:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075713.567936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075714.591727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075714.591844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075714.719790:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.719805:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075714.719810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.719823:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2702 00000100:00080000:3.0:1769075714.719853:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.719862:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075714.719913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612534912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.719975:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.719979:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075714.719981:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.719985:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2702 00000100:00080000:3.0:1769075714.719988:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.719993:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075714.720015:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075714.720027:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612534912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075714.720029:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612535040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.720043:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.720047:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075714.720049:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.720052:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2702 00000100:00080000:3.0:1769075714.720055:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.720057:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075714.720060:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075714.720069:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612535168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.720075:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.720078:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075714.720080:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.720082:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2702 00000100:00080000:3.0:1769075714.720085:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.720087:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075714.720090:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075714.720097:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612535296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075714.720111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075714.720152:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075714] req@ffff8b10037dd500 x1855007612534912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075714.720167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612534912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075714.720181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075714.720185:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075714.720188:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612534912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075714.720210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075714.720226:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075714.720234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075714] req@ffff8b10037ded80 x1855007612535040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075714.720252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612535040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075714.720258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075714.720261:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075714.720263:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075714.720266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075714.720277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075714.720283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075714] req@ffff8b10037de680 x1855007612535168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075714.720291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612535168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075714.720297:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075714.720299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075714.720301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075714.720314:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075714.720326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075714.720332:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075714] req@ffff8b10037dc380 x1855007612535296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075714.720340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612535296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075714.720345:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075714.720347:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075714.720349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535296:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075715.615834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075715.615880:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075716.639704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075716.639765:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075717.663730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075717.663746:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075718.687781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075718.687859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075719.711718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075719.711784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075719.839727:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075719.839749:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075719.839754:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075719.839764:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2707 00000100:00080000:3.0:1769075719.839777:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075719.839784:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075719.839836:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612535424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075719.839859:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075719.839863:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075719.839864:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075719.839868:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2707 00000100:00080000:3.0:1769075719.839887:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075719.839891:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075719.839913:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075719.839928:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612535552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075719.839937:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075719.839940:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075719.839941:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075719.839945:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2707 00000100:00080000:3.0:1769075719.839948:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075719.839951:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075719.839956:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075719.839958:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612535424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075719.839966:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612535680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075719.839974:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075719.839977:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075719.839979:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075719.839982:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2707 00000100:00080000:3.0:1769075719.839985:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075719.839987:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075719.839990:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075719.839999:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612535808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075719.840035:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075719.840068:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff8b10037dca80 x1855007612535424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075719.840084:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612535424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075719.840093:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075719.840097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075719.840100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612535424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075719.840106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075719.840121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075719.840129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff8b10037dc380 x1855007612535552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075719.840138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612535552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075719.840144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075719.840146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075719.840148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075719.840170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075719.840182:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075719.840188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff8b10037de680 x1855007612535680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075719.840196:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612535680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075719.840201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075719.840203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075719.840205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075719.840209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075719.840218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075719.840224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff8b10037ded80 x1855007612535808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075719.840232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612535808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075719.840243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075719.840244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075719.840246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612535808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075720.735792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075720.735981:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075721.759783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075721.759957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075722.783839:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075722.783995:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075723.807778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075723.807879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075724.831802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075724.831924:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075724.959735:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.959747:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075724.959752:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.959763:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2713 00000100:00080000:3.0:1769075724.959777:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.959810:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075724.959854:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612535936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075724.959891:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.959898:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075724.959901:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.959908:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2713 00000100:00080000:3.0:1769075724.959915:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.959923:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075724.959937:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075724.959958:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612535936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075724.959967:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612536064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075724.959993:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.960000:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075724.960004:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.960011:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2713 00000100:00080000:3.0:1769075724.960017:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.960022:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075724.960031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075724.960072:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612536192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075724.960079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075724.960103:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.960108:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075724.960111:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.960118:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2713 00000100:00080000:3.0:1769075724.960123:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.960129:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075724.960136:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075724.960149:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612536320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075724.960167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff8b10037dd500 x1855007612535936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075724.960224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612535936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075724.960240:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075724.960245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075724.960250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612535936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075724.960278:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536064:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075724.960308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075724.960323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff8b10037dd180 x1855007612536064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075724.960337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612536064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075724.960347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075724.960350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075724.960354:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075724.960362:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075724.960380:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075724.960391:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff8b10037de300 x1855007612536192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075724.960404:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612536192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075724.960413:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075724.960416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075724.960419:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075724.960425:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075724.960442:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075724.960452:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff8b10037ddf80 x1855007612536320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075724.960480:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612536320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075724.960508:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075724.960513:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075724.960517:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075725.855808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075725.855960:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075726.879716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075726.879854:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075727.903714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075727.903791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075728.927858:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075728.927976:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075729.951872:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075729.951885:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075730.079881:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.079897:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075730.079905:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.079922:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2718 00000100:00080000:3.0:1769075730.079942:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.079967:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075730.080062:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612536448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.080136:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.080144:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075730.080148:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.080156:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2718 00000100:00080000:3.0:1769075730.080164:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.080174:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075730.080193:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075730.080219:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612536576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075730.080228:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612536448:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075730.080241:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.080263:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075730.080267:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.080274:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2718 00000100:00080000:3.0:1769075730.080281:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.080288:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075730.080297:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075730.080315:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612536704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.080341:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769075730.080344:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075730.080347:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075730.080351:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.080357:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2718 00000100:00080000:3.0:1769075730.080362:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.080368:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075730.080376:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075730.080393:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612536832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075730.080410:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff8b10037dc000 x1855007612536448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075730.080442:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612536448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075730.080477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075730.080486:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075730.080494:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612536448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075730.080533:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075730.080568:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075730.080586:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff8b10037dea00 x1855007612536576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075730.080603:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612536576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075730.080613:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075730.080617:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075730.080621:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075730.080628:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075730.080696:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075730.080740:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff8b10037dd880 x1855007612536704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075730.080764:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612536704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075730.080783:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075730.080787:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075730.080791:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075730.080799:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075730.080832:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075730.080845:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff8b10037df100 x1855007612536832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075730.080859:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612536832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075730.080869:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075730.080872:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075730.080876:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612536832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075730.975816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075730.975900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075731.999839:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075731.999991:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075733.023824:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075733.023884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075734.047788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075734.047856:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075735.071789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075735.071938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075735.199722:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075735.199737:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075735.199744:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075735.199758:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2723 00000100:00080000:3.0:1769075735.199774:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075735.199783:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075735.199841:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612536960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075735.199876:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075735.199880:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075735.199882:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075735.199887:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2723 00000100:00080000:3.0:1769075735.199892:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075735.199897:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075735.199909:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075735.199923:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612537088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075735.199935:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075735.199940:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075735.199942:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075735.199961:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2723 00000100:00080000:3.0:1769075735.199965:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075735.199973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075735.199977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612536960:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769075735.199980:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075735.199998:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612537216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075735.200011:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075735.200015:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075735.200017:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075735.200022:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2723 00000100:00080000:3.0:1769075735.200026:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075735.200030:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075735.200035:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075735.200048:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612537344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075735.200084:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075735.200137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff8b10037ddc00 x1855007612536960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075735.200160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612536960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075735.200174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075735.200179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075735.200184:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612536960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075735.200193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537088:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075735.200216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075735.200229:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff8b10037df800 x1855007612537088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075735.200257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612537088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075735.200266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075735.200270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075735.200273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075735.200309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075735.200342:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075735.200362:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff8b10037dc700 x1855007612537216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075735.200384:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612537216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075735.200398:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075735.200403:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075735.200407:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075735.200414:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075735.200438:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075735.200451:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff8b10037dce00 x1855007612537344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075735.200464:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612537344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075735.200473:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075735.200476:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075735.200479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075736.095766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075736.095912:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075737.119878:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075737.119888:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075738.143806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075738.143985:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075739.167738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075739.167855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075740.192058:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075740.192066:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075740.319748:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075740.319762:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075740.319768:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075740.319782:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2728 00000100:00080000:3.0:1769075740.319802:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075740.319812:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075740.319878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612537472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075740.319915:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075740.319920:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075740.319922:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075740.319926:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2728 00000100:00080000:3.0:1769075740.319930:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075740.319934:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075740.319957:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075740.319974:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612537600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075740.319986:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612537472:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075740.319989:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075740.319997:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075740.319999:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075740.320003:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2728 00000100:00080000:3.0:1769075740.320017:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075740.320020:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075740.320025:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075740.320037:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612537728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075740.320046:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075740.320049:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075740.320051:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075740.320055:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2728 00000100:00080000:3.0:1769075740.320058:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075740.320061:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075740.320065:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075740.320073:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612537856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075740.320115:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075740.320179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff8b10037df480 x1855007612537472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075740.320198:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612537472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075740.320210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075740.320214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075740.320218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612537472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075740.320245:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537600:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075740.320266:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075740.320295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff8b10037dce00 x1855007612537600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075740.320311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612537600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075740.320330:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075740.320334:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075740.320338:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075740.320346:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075740.320386:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075740.320424:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff8b10037dc700 x1855007612537728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075740.320460:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612537728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075740.320491:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075740.320495:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075740.320511:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075740.320519:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075740.320577:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075740.320589:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff8b10037df800 x1855007612537856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075740.320600:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612537856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075740.320607:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075740.320610:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075740.320614:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612537856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075741.215796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075741.216015:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075742.239865:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075742.239945:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075743.263754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075743.263812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075744.287899:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075744.287949:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075745.311768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075745.311908:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075745.439766:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075745.439783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075745.439791:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075745.439812:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2733 00000100:00080000:3.0:1769075745.439836:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075745.439854:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075745.439926:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612537984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075745.439981:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075745.440003:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075745.440008:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075745.440016:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2733 00000100:00080000:3.0:1769075745.440023:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075745.440031:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075745.440043:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612537984:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769075745.440051:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075745.440077:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612538112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075745.440098:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075745.440104:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075745.440108:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075745.440115:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2733 00000100:00080000:3.0:1769075745.440149:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075745.440155:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075745.440163:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075745.440178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075745.440188:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612538240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075745.440204:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075745.440209:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075745.440212:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075745.440217:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2733 00000100:00080000:3.0:1769075745.440223:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075745.440229:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075745.440238:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075745.440254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff8b10037ddc00 x1855007612537984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075745.440255:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612538368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075745.440323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612537984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075745.440348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075745.440357:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075745.440366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612537984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075745.440415:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075745.440491:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075745.440524:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff8b10037df100 x1855007612538112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075745.440555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612538112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075745.440575:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075745.440583:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075745.440590:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075745.440600:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075745.440685:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075745.440708:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff8b10037dd880 x1855007612538240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075745.440738:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612538240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075745.440751:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075745.440758:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075745.440767:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075745.440793:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075745.440823:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075745.440838:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff8b10037dea00 x1855007612538368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075745.440855:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612538368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075745.440867:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075745.440873:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075745.440880:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075746.335773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075746.335939:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075747.359706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075747.359881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075748.383773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075748.383902:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075749.407740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075749.407969:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075750.431753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075750.431837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075750.559833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075750.559859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075750.559865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075750.559875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2738 00000100:00080000:3.0:1769075750.559896:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075750.559904:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075750.559962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612538496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075750.559993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075750.559996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075750.559998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075750.560001:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2738 00000100:00080000:3.0:1769075750.560003:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075750.560006:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075750.560015:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075750.560028:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612538624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075750.560036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075750.560039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075750.560040:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075750.560044:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2738 00000100:00080000:3.0:1769075750.560047:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075750.560050:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075750.560066:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075750.560074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612538752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075750.560086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075750.560089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612538496:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075750.560091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075750.560098:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075750.560113:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2738 00000100:00080000:3.0:1769075750.560117:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075750.560119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075750.560122:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075750.560149:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612538880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075750.560233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075750.560302:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff8b10037dc000 x1855007612538496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075750.560328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612538496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075750.560342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075750.560348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075750.560412:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612538496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075750.560420:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538624:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075750.560439:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075750.560483:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff8b10037ddf80 x1855007612538624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075750.560536:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612538624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075750.560546:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075750.560551:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075750.560554:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075750.560580:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075750.560634:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075750.560646:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff8b10037de300 x1855007612538752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075750.560658:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612538752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075750.560666:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075750.560708:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075750.560713:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075750.560723:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075750.560753:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075750.560795:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff8b10037dd180 x1855007612538880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075750.560808:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612538880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075750.560817:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075750.560820:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075750.560822:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612538880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075751.455830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075751.455918:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075752.479758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075752.479829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075753.503731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075753.503821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075754.527735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075754.527821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075755.551716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075755.551776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075755.679732:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.679744:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075755.679749:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.679761:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2743 00000100:00080000:3.0:1769075755.679775:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.679794:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075755.679837:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612539008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.679869:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.679875:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075755.679876:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.679879:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2743 00000100:00080000:3.0:1769075755.679882:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.679885:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075755.679894:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075755.679905:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612539136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.679914:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075755.679917:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612539008:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075755.679918:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075755.679921:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.679928:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2743 00000100:00080000:3.0:1769075755.679931:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.679933:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075755.679936:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075755.679946:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612539264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.679953:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.679956:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075755.679957:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.679960:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2743 00000100:00080000:3.0:1769075755.679962:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.679964:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075755.679969:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075755.679976:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612539392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075755.680016:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075755.680055:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff8b10037dd500 x1855007612539008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075755.680070:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612539008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075755.680080:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075755.680083:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075755.680087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612539008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075755.680110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769075755.680169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075755.680186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff8b10037ded80 x1855007612539136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075755.680203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612539136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075755.680214:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075755.680219:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075755.680223:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769075755.680231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075755.680254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075755.680266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff8b10037de680 x1855007612539264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075755.680365:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612539264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075755.680375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075755.680379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075755.680383:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075755.680390:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769075755.680411:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769075755.680420:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff8b10037dc380 x1855007612539392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075755.680428:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612539392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075755.680434:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075755.680437:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075755.680439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075756.575747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075756.575823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075757.599720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075757.599799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075758.623774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075758.623823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075759.647720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075759.647789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075760.671731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075760.671860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075760.799736:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075760.799746:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075760.799750:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075760.799761:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2748 00000100:00080000:2.0:1769075760.799772:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075760.799778:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075760.799814:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612539520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075760.799846:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075760.799850:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075760.799851:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075760.799856:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2748 00000100:00080000:2.0:1769075760.799859:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075760.799863:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075760.799872:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075760.799885:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612539648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075760.799896:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075760.799899:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075760.799901:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075760.799906:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2748 00000100:00080000:2.0:1769075760.799910:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075760.799914:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075760.799919:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075760.799928:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612539520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075760.799934:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612539776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075760.799960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075760.799964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075760.799966:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075760.799971:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2748 00000100:00080000:2.0:1769075760.799975:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075760.799979:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075760.799984:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075760.799997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612539904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075760.800028:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075760.800074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff8b1005d59880 x1855007612539520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075760.800093:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612539520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075760.800102:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075760.800107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075760.800110:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612539520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075760.800117:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075760.800135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075760.800144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff8b1005d59500 x1855007612539648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075760.800153:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612539648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075760.800159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075760.800162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075760.800172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075760.800215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075760.800236:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075760.800244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff8b1005d5ad80 x1855007612539776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075760.800253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612539776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075760.800259:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075760.800262:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075760.800264:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075760.800268:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075760.800280:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075760.800286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff8b1005d5aa00 x1855007612539904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075760.800294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612539904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075760.800300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075760.800302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075760.800303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612539904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075761.695738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075761.695817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075762.719740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075762.719834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075763.743763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075763.743819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075764.767728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075764.767795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075765.791761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075765.791826:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075765.919702:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075765.919711:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075765.919715:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075765.919723:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2754 00000100:00080000:2.0:1769075765.919733:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075765.919739:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075765.919770:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612540032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075765.919794:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075765.919797:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075765.919799:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075765.919803:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2754 00000100:00080000:2.0:1769075765.919807:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075765.919811:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075765.919821:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075765.919834:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612540160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075765.919845:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075765.919850:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075765.919853:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075765.919858:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2754 00000100:00080000:2.0:1769075765.919863:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075765.919867:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075765.919872:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075765.919882:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612540288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075765.919889:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075765.919891:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075765.919892:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075765.919895:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2754 00000100:00080000:2.0:1769075765.919897:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075765.919900:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075765.919903:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075765.919916:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612540416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075765.919919:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612540032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075765.920059:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075765.920095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075765.920159:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075766.815697:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075766.815804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075767.839760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075767.839818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075768.863796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075768.863868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075768.991967:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075768.992019:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075768.992042:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075768.992106:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075768.992120:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075768.992133:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075768.992200:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075768] req@ffff8b1005d5ad80 x1855007612540416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075768.992234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612540416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075768.992243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075768.992248:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075768.992251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075768.992257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075768] req@ffff8b1005d58380 x1855007612540032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075768.992265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612540032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075768.992270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075768.992273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075768.992275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612540032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075768.992280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075768] req@ffff8b1005d5b100 x1855007612540160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075768.992288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612540160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075768.992292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075768.992295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075768.992298:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075768.992303:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075768] req@ffff8b1005d5aa00 x1855007612540288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075768.992311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612540288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075768.992315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075768.992318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075768.992320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075769.887715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075769.887888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075770.911739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075770.911841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075771.039796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075771.039809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075771.039813:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075771.039824:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2759 00000100:00080000:0.0:1769075771.039837:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075771.039844:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075771.039883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612540544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075771.039914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075771.039917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075771.039919:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075771.039922:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2759 00000100:00080000:0.0:1769075771.039925:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075771.039927:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075771.039936:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075771.039946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612540672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075771.039958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075771.039961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075771.039963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075771.039966:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2759 00000100:00080000:0.0:1769075771.039968:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075771.039971:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075771.039974:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075771.039982:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612540800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075771.039991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075771.039995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075771.039997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075771.040002:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2759 00000100:00100000:1.0:1769075771.040005:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612540544:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075771.040006:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075771.040029:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075771.040036:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075771.040062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612540928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075771.040116:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075771.040165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff8b1006026d80 x1855007612540544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075771.040187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612540544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075771.040200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075771.040205:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075771.040211:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612540544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075771.040220:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075771.040248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075771.040263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff8b1006027b80 x1855007612540672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075771.040285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612540672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075771.040294:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075771.040298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075771.040301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075771.040335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075771.040360:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075771.040373:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff8b1006025f80 x1855007612540800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075771.040385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612540800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075771.040393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075771.040397:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075771.040400:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075771.040406:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075771.040425:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075771.040437:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075771/real 1769075771] req@ffff8b1006024a80 x1855007612540928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075771.040450:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612540928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075771.040465:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075771.040468:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075771.040471:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612540928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075771.935783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075771.935886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075772.959804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075772.959844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075773.983746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075773.983811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075775.007730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075775.007822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075776.031740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075776.031822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075776.159706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075776.159715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075776.159719:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075776.159728:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2764 00000100:00080000:0.0:1769075776.159737:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075776.159743:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075776.159774:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612541056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075776.159801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075776.159804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075776.159806:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075776.159809:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2764 00000100:00080000:0.0:1769075776.159811:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075776.159813:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075776.159820:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075776.159829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612541184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075776.159835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075776.159838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075776.159839:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075776.159841:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2764 00000100:00080000:0.0:1769075776.159843:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075776.159845:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075776.159848:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075776.159856:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612541312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075776.159867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075776.159870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075776.159871:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075776.159874:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2764 00000100:00080000:0.0:1769075776.159876:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075776.159879:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075776.159883:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075776.159884:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612541056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075776.159892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612541440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075776.159967:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075776.160000:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff8b1006024e00 x1855007612541056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075776.160014:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612541056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075776.160022:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075776.160026:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075776.160028:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612541056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075776.160034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541184:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075776.160050:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075776.160058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff8b1006024380 x1855007612541184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075776.160066:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612541184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075776.160071:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075776.160073:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075776.160075:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075776.160080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075776.160091:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075776.160097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff8b1006027480 x1855007612541312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075776.160106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612541312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075776.160111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075776.160113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075776.160115:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075776.160134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075776.160144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075776.160150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff8b1006024700 x1855007612541440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075776.160158:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612541440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075776.160163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075776.160165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075776.160167:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075777.055716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075777.055794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075778.079743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075778.079819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075779.103756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075779.103833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075780.127784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075780.127799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075781.151766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075781.151811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075781.279775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075781.279784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075781.279789:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075781.279798:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2769 00000100:00080000:0.0:1769075781.279810:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075781.279817:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075781.279880:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612541568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075781.279913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075781.279917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075781.279919:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075781.279925:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2769 00000100:00080000:0.0:1769075781.279930:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075781.279934:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075781.279944:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075781.279965:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612541696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075781.279965:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612541568:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075781.279988:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075781.279993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075781.279996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075781.280000:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2769 00000100:00080000:0.0:1769075781.280006:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075781.280009:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075781.280014:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075781.280025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612541824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075781.280034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075781.280041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1769075781.280041:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075781.280047:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075781.280054:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2769 00000100:00080000:0.0:1769075781.280060:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075781.280064:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075781.280069:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075781.280084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612541952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075781.280090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075781] req@ffff8b1006024700 x1855007612541568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075781.280216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612541568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075781.280226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075781.280230:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075781.280234:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612541568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075781.280255:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075781.280277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075781.280290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075781] req@ffff8b1006027480 x1855007612541696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075781.280301:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612541696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075781.280314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075781.280317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075781.280320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075781.280326:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075781.280340:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075781.280479:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075781] req@ffff8b1006024380 x1855007612541824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075781.280492:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612541824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075781.280500:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075781.280503:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075781.280506:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075781.280511:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075781.280534:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075781.280598:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075781] req@ffff8b1006024e00 x1855007612541952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075781.280615:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612541952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075781.280624:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075781.280628:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075781.280632:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612541952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075782.175744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075782.175843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075783.199746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075783.199843:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075784.223763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075784.223817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075785.247809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075785.247947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075786.271708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075786.271812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075786.399776:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075786.399788:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075786.399793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075786.399816:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2774 00000100:00080000:3.0:1769075786.399829:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075786.399837:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075786.399883:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612542080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075786.399920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075786.399925:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075786.399927:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075786.399932:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2774 00000100:00080000:3.0:1769075786.399937:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075786.399942:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075786.399953:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075786.399969:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612542208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075786.399977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612542080:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075786.399981:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075786.399988:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075786.399989:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075786.399993:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2774 00000100:00080000:3.0:1769075786.399996:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075786.400001:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075786.400005:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075786.400015:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612542336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075786.400023:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075786.400026:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075786.400028:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075786.400030:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2774 00000100:00080000:3.0:1769075786.400033:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075786.400036:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075786.400039:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075786.400048:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612542464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075786.400063:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075786.400102:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff8b10037dca80 x1855007612542080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075786.400120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612542080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075786.400129:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075786.400133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075786.400137:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612542080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075786.400161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075786.400177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075786.400186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff8b10037dc380 x1855007612542208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075786.400195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612542208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075786.400202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075786.400204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075786.400206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075786.400211:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075786.400224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075786.400230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff8b10037de680 x1855007612542336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075786.400239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612542336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075786.400249:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075786.400252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075786.400271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075786.400276:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075786.400292:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075786.400301:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff8b10037ded80 x1855007612542464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075786.400311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612542464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075786.400318:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075786.400320:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075786.400323:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542464:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075787.295871:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075787.295874:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075788.319767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075788.319987:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075789.343811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075789.343883:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075790.367786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075790.367858:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075791.391802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075791.391888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075791.519761:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.519774:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075791.519779:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.519790:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2779 00000100:00080000:3.0:1769075791.519801:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.519808:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075791.519846:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612542592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.519877:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.519881:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075791.519883:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.519887:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2779 00000100:00080000:3.0:1769075791.519890:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.519893:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075791.519901:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075791.519912:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612542720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.519922:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.519926:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075791.519928:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.519932:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2779 00000100:00080000:3.0:1769075791.519936:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.519940:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075791.519944:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075791.519951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612542848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.519962:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.519965:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075791.519969:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075791.519977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612542592:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075791.519977:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2779 00000100:00080000:3.0:1769075791.519988:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.519997:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075791.520002:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075791.520014:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612542976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075791.520091:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075791.520141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff8b10037dd500 x1855007612542592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075791.520159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612542592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075791.520170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075791.520174:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075791.520178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612542592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075791.520185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542720:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075791.520205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075791.520214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff8b10037dd180 x1855007612542720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075791.520223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612542720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075791.520229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075791.520231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075791.520233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075791.520267:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075791.520290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075791.520303:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff8b10037de300 x1855007612542848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075791.520315:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612542848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075791.520325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075791.520329:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075791.520332:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075791.520339:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075791.520358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075791.520370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff8b10037ddf80 x1855007612542976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075791.520383:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612542976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075791.520390:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075791.520393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075791.520397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612542976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075792.415827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075792.415911:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075793.439804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075793.439940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075794.463825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075794.463827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075795.487747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075795.487834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075796.511729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075796.511813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075796.639830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075796.639843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075796.639849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075796.639861:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2784 00000100:00080000:0.0:1769075796.639875:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075796.639912:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075796.639972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612543104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075796.640021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075796.640026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075796.640029:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075796.640035:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2784 00000100:00080000:0.0:1769075796.640041:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075796.640046:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075796.640058:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075796.640077:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612543232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075796.640092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075796.640096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075796.640099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075796.640104:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2784 00000100:00080000:0.0:1769075796.640109:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075796.640114:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075796.640121:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075796.640135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612543104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075796.640137:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612543360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075796.640161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075796.640165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075796.640168:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075796.640174:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2784 00000100:00080000:0.0:1769075796.640179:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075796.640183:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075796.640189:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075796.640203:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612543488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075796.640237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075796.640284:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff8b1006024a80 x1855007612543104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075796.640308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612543104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075796.640322:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075796.640328:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075796.640334:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612543104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075796.640343:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075796.640369:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075796.640383:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff8b1006025f80 x1855007612543232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075796.640416:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612543232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075796.640436:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075796.640441:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075796.640445:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075796.640475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075796.640500:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075796.640513:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff8b1006027b80 x1855007612543360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075796.640528:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612543360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075796.640539:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075796.640542:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075796.640546:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075796.640553:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075796.640573:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075796.640614:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff8b1006026d80 x1855007612543488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075796.640631:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612543488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075796.640682:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075796.640687:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075796.640692:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075797.535794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075797.535947:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075798.559794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075798.559971:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075799.583793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075799.583946:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075800.607822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075800.607927:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075801.631798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075801.631827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075801.759787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075801.759795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075801.759799:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075801.759807:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2789 00000100:00080000:0.0:1769075801.759818:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075801.759825:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075801.759860:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612543616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075801.759885:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075801.759888:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075801.759889:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075801.759892:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2789 00000100:00080000:0.0:1769075801.759894:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075801.759897:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075801.759903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075801.759911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612543744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075801.759918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075801.759920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075801.759921:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075801.759924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2789 00000100:00080000:0.0:1769075801.759925:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075801.759927:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075801.759930:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075801.759937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612543872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075801.759944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075801.759946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075801.759947:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075801.759950:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2789 00000100:00080000:0.0:1769075801.759953:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075801.759956:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075801.759959:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075801.759967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612544000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075801.759968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612543616:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769075801.760027:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075801.760057:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff8b1006026300 x1855007612543616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075801.760071:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612543616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075801.760079:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075801.760082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075801.760085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612543616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075801.760090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543744:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075801.760103:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075801.760111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff8b1006024000 x1855007612543744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075801.760119:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612543744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075801.760124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075801.760126:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075801.760128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075801.760132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075801.760143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075801.760149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff8b1006025880 x1855007612543872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075801.760157:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612543872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075801.760162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075801.760164:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075801.760166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612543872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075801.760187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075801.760208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075801.760220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff8b1006026a00 x1855007612544000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075801.760232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612544000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075801.760241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075801.760244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075801.760248:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075802.655763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075802.655865:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075803.679731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075803.679887:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075804.703737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075804.703805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075805.727736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075805.727817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075806.751798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075806.751914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075806.879757:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075806.879768:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075806.879773:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075806.879783:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2794 00000100:00080000:3.0:1769075806.879794:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075806.879801:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075806.879841:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612544128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075806.879870:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075806.879875:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075806.879877:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075806.879881:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2794 00000100:00080000:3.0:1769075806.879885:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075806.879890:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075806.879901:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075806.879913:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612544128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075806.879918:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612544256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075806.879939:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075806.879944:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075806.879946:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075806.879950:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2794 00000100:00080000:3.0:1769075806.879955:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075806.879959:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075806.879964:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075806.879978:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612544384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075806.879979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075806.879992:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075806.879996:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075806.879998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075806.880004:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2794 00000100:00080000:3.0:1769075806.880008:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075806.880013:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075806.880019:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075806.880024:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff8b10037dc000 x1855007612544128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075806.880033:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612544512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075806.880044:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612544128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075806.880055:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075806.880059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075806.880062:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612544128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075806.880083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075806.880101:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075806.880110:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff8b10037dea00 x1855007612544256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075806.880119:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612544256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075806.880125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075806.880127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075806.880129:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075806.880133:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075806.880144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075806.880151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff8b10037dd880 x1855007612544384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075806.880158:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612544384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075806.880164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075806.880166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075806.880168:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075806.880172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075806.880183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075806.880189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff8b10037df100 x1855007612544512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075806.880197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612544512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075806.880202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075806.880204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075806.880206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075807.775737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075807.775820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075808.799885:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075808.799907:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075809.823743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075809.823815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075810.847708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075810.847796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075811.871750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075811.871880:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075811.999787:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.999800:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075811.999807:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.999820:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2800 00000100:00080000:3.0:1769075811.999861:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.999867:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075811.999906:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612544640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075811.999937:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.999940:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075811.999942:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.999946:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2800 00000100:00080000:3.0:1769075811.999949:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.999952:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075811.999960:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075811.999970:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612544768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075811.999979:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.999983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075811.999985:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.999988:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2800 00000100:00080000:3.0:1769075811.999992:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.999995:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075812.000000:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075812.000005:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612544640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075812.000013:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612544896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075812.000031:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075812.000034:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075812.000035:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075812.000038:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2800 00000100:00080000:3.0:1769075812.000041:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075812.000045:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075812.000049:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075812.000058:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612545024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075812.000079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075812.000118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075812] req@ffff8b10037ddc00 x1855007612544640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075812.000164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612544640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075812.000178:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075812.000184:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075812.000189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612544640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075812.000199:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075812.000227:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075812.000242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075812] req@ffff8b10037df800 x1855007612544768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075812.000260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612544768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075812.000270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075812.000274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075812.000278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075812.000309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075812.000400:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075812.000431:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075812] req@ffff8b10037dc700 x1855007612544896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075812.000448:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612544896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075812.000459:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075812.000463:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075812.000467:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612544896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075812.000475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075812.000498:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075812.000510:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075812] req@ffff8b10037dce00 x1855007612545024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075812.000540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612545024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075812.000552:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075812.000555:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075812.000558:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075812.895800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075812.895870:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075813.919739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075813.919840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075814.943749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075814.943806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075815.967817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075815.967851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075816.991722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075816.991778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075817.119729:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.119737:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075817.119742:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.119750:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2805 00000100:00080000:3.0:1769075817.119760:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.119767:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075817.119801:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612545152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.119825:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.119829:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.119831:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.119840:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2805 00000100:00080000:3.0:1769075817.119844:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.119849:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075817.119858:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.119873:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612545280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075817.119886:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612545152:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075817.119894:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.119897:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075817.119899:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.119902:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2805 00000100:00080000:3.0:1769075817.119906:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075817.119909:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075817.119914:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.119924:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612545408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.119932:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075817.119934:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075817.119936:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075817.119939:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2805 00000100:00080000:3.0:1769075817.119942:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075817.119945:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075817.119946:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075817.119950:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075817.119958:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612545536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075817.119979:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff8b10037df480 x1855007612545152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075817.119996:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612545152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075817.120005:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075817.120009:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075817.120013:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612545152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075817.120036:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545280:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075817.120059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075817.120072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff8b10037dce00 x1855007612545280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075817.120085:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612545280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075817.120093:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075817.120097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075817.120100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075817.120109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075817.120146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075817.120162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff8b10037dc700 x1855007612545408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075817.120177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612545408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075817.120202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075817.120218:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075817.120222:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075817.120228:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075817.120265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075817.120280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff8b10037df800 x1855007612545536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075817.120302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612545536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075817.120312:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075817.120316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075817.120319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075818.015766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075818.015835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075819.039774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075819.039883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075820.063783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075820.063875:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075821.087706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075821.087762:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075822.111810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075822.111946:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075822.239796:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075822.239807:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075822.239812:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075822.239823:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2810 00000100:00080000:2.0:1769075822.239835:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075822.239843:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075822.239886:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612545664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075822.239920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075822.239924:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075822.239926:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075822.239930:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2810 00000100:00080000:2.0:1769075822.239934:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075822.239938:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075822.239949:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075822.239962:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612545792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075822.239976:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075822.239983:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612545664:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769075822.239983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075822.239988:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075822.239999:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2810 00000100:00080000:2.0:1769075822.240003:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075822.240007:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075822.240012:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075822.240026:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612545920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075822.240041:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075822.240046:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075822.240049:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075822.240057:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2810 00000100:00080000:2.0:1769075822.240064:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075822.240070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769075822.240078:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075822.240089:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075822.240113:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612546048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075822.240120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff8b1005d59500 x1855007612545664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075822.240201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612545664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075822.240217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075822.240224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075822.240229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612545664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075822.240260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075822.240291:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075822.240307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff8b1005d59880 x1855007612545792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075822.240352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612545792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075822.240369:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075822.240377:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075822.240383:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075822.240396:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075822.240434:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075822.240456:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff8b1005d5bb80 x1855007612545920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075822.240481:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612545920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075822.240506:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075822.240512:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075822.240518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612545920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075822.240529:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075822.240566:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075822.240587:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff8b1005d58e00 x1855007612546048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075822.240609:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612546048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075822.240640:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075822.240645:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075822.240650:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075823.135774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075823.135802:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075824.159800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075824.159968:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075825.183736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075825.183828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075826.207734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075826.207807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075827.231766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075827.231849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075827.359770:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075827.359780:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075827.359785:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075827.359794:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2815 00000100:00080000:2.0:1769075827.359806:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075827.359813:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075827.359850:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612546176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075827.359879:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075827.359883:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075827.359885:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075827.359889:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2815 00000100:00080000:2.0:1769075827.359892:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075827.359896:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075827.359904:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075827.359914:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612546304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075827.359923:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075827.359926:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075827.359927:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075827.359930:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2815 00000100:00080000:2.0:1769075827.359932:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075827.359935:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075827.359938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075827.359945:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612546432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075827.359954:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075827.359957:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075827.359958:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075827.359962:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2815 00000100:00080000:2.0:1769075827.359965:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075827.359968:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075827.359973:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075827.359980:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612546560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075827.359988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612546176:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769075827.360106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075827.360155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff8b1005d5b800 x1855007612546176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075827.360176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612546176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075827.360187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075827.360192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075827.360196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612546176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075827.360205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546304:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075827.360232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075827.360246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff8b1005d5b480 x1855007612546304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075827.360255:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612546304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075827.360261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075827.360279:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075827.360282:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075827.360288:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075827.360306:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075827.360314:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff8b1005d59180 x1855007612546432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075827.360327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612546432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075827.360337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075827.360341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075827.360359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075827.360387:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075827.360409:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075827.360420:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff8b1005d5a300 x1855007612546560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075827.360433:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612546560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075827.360442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075827.360445:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075827.360449:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075828.255812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075828.255937:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075829.279751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075829.279859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075830.303765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075830.303776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075831.327825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075831.327941:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075832.351731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075832.351825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075832.479793:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.479806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075832.479813:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.479824:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2820 00000100:00080000:0.0:1769075832.479838:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.479846:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075832.479889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612546688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075832.479927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.479931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075832.479934:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.479938:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2820 00000100:00080000:0.0:1769075832.479942:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.479947:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075832.479957:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075832.479970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612546816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075832.480019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.480024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075832.480027:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.480032:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2820 00000100:00080000:0.0:1769075832.480040:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.480045:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075832.480053:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075832.480069:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612546688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075832.480071:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612546944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075832.480110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.480115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075832.480117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.480123:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2820 00000100:00080000:0.0:1769075832.480129:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.480133:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075832.480140:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075832.480156:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612547072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075832.480226:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075832.480272:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075832.480300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075833.375800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075833.375874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075834.399828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075834.399998:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075835.423802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075835.424051:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075835.552270:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769075835.552341:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075835.552368:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075835.552437:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075835.552459:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769075835.552480:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769075835.552494:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075835] req@ffff8b1006027800 x1855007612546688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075835.552565:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612546688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075835.552591:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075835.552602:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075835.552611:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612546688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075835.552626:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075835] req@ffff8b1006027100 x1855007612546816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075835.552724:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612546816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075835.552746:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075835.552755:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075835.552763:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075835.552808:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075835] req@ffff8b1006026680 x1855007612546944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075835.552846:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612546944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075835.553041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075835.553050:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075835.553057:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612546944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075835.553072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075835] req@ffff8b1006025180 x1855007612547072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075835.553098:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612547072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075835.553115:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075835.553123:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075835.553130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075836.447771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075836.447881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075837.471836:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075837.471870:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075837.599833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075837.599849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075837.599854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075837.599867:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2825 00000100:00080000:3.0:1769075837.599881:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075837.599890:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075837.599971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612547200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075837.600021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075837.600029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075837.600033:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075837.600042:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2825 00000100:00080000:3.0:1769075837.600051:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075837.600058:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075837.600074:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075837.600098:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612547328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075837.600117:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612547200:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075837.600119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075837.600126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075837.600129:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075837.600136:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2825 00000100:00080000:3.0:1769075837.600143:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075837.600150:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075837.600159:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075837.600176:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612547456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075837.600192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075837.600198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075837.600201:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075837.600209:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2825 00000100:00080000:3.0:1769075837.600215:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075837.600222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075837.600222:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075837.600230:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075837.600247:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612547584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075837.600277:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff8b10037ddc00 x1855007612547200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075837.600312:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612547200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075837.600334:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075837.600343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075837.600350:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612547200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075837.600391:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075837.600438:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075837.600465:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff8b10037df100 x1855007612547328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075837.600490:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612547328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075837.600504:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075837.600510:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075837.600516:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075837.600529:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075837.600568:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075837.600603:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff8b10037dd880 x1855007612547456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075837.600629:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612547456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075837.600696:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075837.600704:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075837.600711:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075837.600724:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075837.600768:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075837.600792:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff8b10037dea00 x1855007612547584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075837.600816:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612547584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075837.600832:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075837.600839:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075837.600846:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075838.495742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075838.495825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075839.519798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075839.519978:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075840.543710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075840.543833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075841.567721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075841.567888:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075842.591706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075842.591890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075842.719827:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075842.719838:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075842.719844:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075842.719854:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2830 00000100:00080000:3.0:1769075842.719866:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075842.719873:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075842.719913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612547712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075842.719946:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075842.719952:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075842.719954:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075842.719960:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2830 00000100:00080000:3.0:1769075842.719965:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075842.719972:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075842.719983:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075842.719999:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612547840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075842.720014:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075842.720019:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075842.720021:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075842.720026:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2830 00000100:00100000:1.0:1769075842.720027:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612547712:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075842.720032:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075842.720036:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075842.720043:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075842.720073:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612547968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075842.720088:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075842.720093:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075842.720096:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075842.720101:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2830 00000100:00080000:3.0:1769075842.720106:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075842.720111:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075842.720132:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075842.720145:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612548096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075842.720147:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075842.720216:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff8b10037dc000 x1855007612547712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075842.720241:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612547712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075842.720251:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075842.720255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075842.720259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612547712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075842.720266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547840:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075842.720310:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075842.720327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff8b10037ddf80 x1855007612547840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075842.720342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612547840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075842.720351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075842.720355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075842.720359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075842.720402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075842.720455:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075842.720474:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff8b10037de300 x1855007612547968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075842.720492:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612547968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075842.720521:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075842.720527:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075842.720534:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612547968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075842.720544:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075842.720578:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075842.720597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff8b10037dd180 x1855007612548096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075842.720619:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612548096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075842.720709:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075842.720713:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075842.720717:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075843.615811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075843.615837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075844.639827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075844.640016:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075845.663791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075845.663839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075846.687844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075846.687956:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075847.711740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075847.711808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075847.839811:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.839823:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075847.839829:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.839839:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2835 00000100:00080000:3.0:1769075847.839850:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.839857:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075847.839893:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612548224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.839916:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.839920:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075847.839922:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.839924:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2835 00000100:00080000:3.0:1769075847.839927:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.839931:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075847.839938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075847.839948:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612548352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.839957:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075847.839959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612548224:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075847.839971:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075847.839974:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.839978:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2835 00000100:00080000:3.0:1769075847.839982:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.839985:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075847.839988:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075847.839999:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612548480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.840005:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.840008:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075847.840009:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.840012:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2835 00000100:00080000:3.0:1769075847.840014:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.840017:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769075847.840023:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769075847.840024:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075847.840039:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612548608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075847.840081:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075847] req@ffff8b10037dd500 x1855007612548224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075847.840100:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612548224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075847.840110:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075847.840115:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075847.840119:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612548224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075847.840145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075847.840173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075847.840187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075847] req@ffff8b10037ded80 x1855007612548352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075847.840197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612548352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075847.840204:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075847.840206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075847.840209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075847.840215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075847.840240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075847.840249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075847] req@ffff8b10037de680 x1855007612548480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075847.840258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612548480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075847.840264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075847.840267:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075847.840269:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075847.840274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075847.840287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075847.840293:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075847] req@ffff8b10037dc380 x1855007612548608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075847.840302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612548608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075847.840308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075847.840310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075847.840312:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075848.735769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075848.735913:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075849.759740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075849.759806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075850.783761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075850.783830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075851.807800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075851.807818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075852.831754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075852.831870:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075852.959765:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075852.959773:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075852.959778:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075852.959786:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2841 00000100:00080000:2.0:1769075852.959796:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075852.959802:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769075852.959834:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612548736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075852.959856:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075852.959859:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075852.959861:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075852.959864:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2841 00000100:00080000:2.0:1769075852.959868:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075852.959871:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769075852.959879:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075852.959887:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612548736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075852.959896:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612548864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075852.959910:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075852.959915:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075852.959917:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075852.959923:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2841 00000100:00080000:2.0:1769075852.959938:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075852.959944:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769075852.959948:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769075852.959955:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075852.959971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612548992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075852.959980:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff8b1005d58000 x1855007612548736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075852.959990:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075852.959996:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075852.959998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075852.960002:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2841 00000100:00100000:3.0:1769075852.960022:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612548736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075852.960024:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075852.960032:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:3.0:1769075852.960038:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769075852.960039:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075852.960048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769075852.960055:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612549120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075852.960055:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612548736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075852.960086:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075852.960169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075852.960180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff8b1005d58a80 x1855007612548864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075852.960191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612548864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075852.960197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075852.960216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075852.960219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075852.960223:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075852.960238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075852.960245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff8b1005d5a300 x1855007612548992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075852.960252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612548992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075852.960258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075852.960259:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075852.960261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612548992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075852.960265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075852.960276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075852.960282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff8b1005d59180 x1855007612549120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075852.960290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612549120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075852.960402:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075852.960406:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075852.960409:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075853.855821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075853.855853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075854.879701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075854.879800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075855.903776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075855.903866:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075856.927722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075856.927790:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075857.951766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075857.951870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075858.079752:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.079762:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075858.079767:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.079776:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2846 00000100:00080000:3.0:1769075858.079786:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.079792:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075858.079827:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612549248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.079856:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.079860:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075858.079862:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.079865:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2846 00000100:00080000:3.0:1769075858.079868:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.079871:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075858.079879:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075858.079891:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612549376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.079901:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.079904:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075858.079905:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.079908:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2846 00000100:00080000:3.0:1769075858.079911:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.079918:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075858.079920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612549248:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769075858.079923:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075858.079936:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612549504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.079945:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.079948:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075858.079950:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.079953:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2846 00000100:00080000:3.0:1769075858.079956:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.079959:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075858.079962:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075858.079971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612549632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075858.079989:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075858.080023:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff8b10037dca80 x1855007612549248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.080040:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612549248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.080049:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.080052:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075858.080056:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612549248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075858.080062:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075858.080078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075858.080087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff8b10037dc380 x1855007612549376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.080097:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612549376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.080103:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.080105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075858.080108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075858.080127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075858.080140:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075858.080151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff8b10037de680 x1855007612549504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.080167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612549504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.080185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.080190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075858.080194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075858.080202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075858.080225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075858.080236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff8b10037ded80 x1855007612549632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.080246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612549632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.080252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.080255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075858.080257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075858.975806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075858.975822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075859.999770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075859.999834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075861.023770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075861.023935:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075862.047764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075862.047850:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075863.071796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075863.071950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075863.199779:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075863.199793:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075863.199798:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075863.199810:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2851 00000100:00080000:3.0:1769075863.199824:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075863.199833:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075863.199883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612549760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075863.199935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075863.199940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075863.199943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075863.199948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2851 00000100:00080000:3.0:1769075863.199954:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075863.199959:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075863.199970:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075863.199985:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612549888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075863.200000:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075863.200004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075863.200007:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075863.200013:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2851 00000100:00080000:3.0:1769075863.200018:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075863.200022:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075863.200029:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075863.200056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612549760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075863.200057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612550016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075863.200084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075863.200088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075863.200091:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075863.200096:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2851 00000100:00080000:3.0:1769075863.200101:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075863.200105:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075863.200112:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075863.200128:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612550144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075863.200160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075863.200211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff8b10037dd500 x1855007612549760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075863.200247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612549760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.200261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075863.200266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075863.200272:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612549760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075863.200282:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075863.200308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075863.200323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff8b10037dd180 x1855007612549888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075863.200339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612549888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.200349:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075863.200353:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075863.200357:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612549888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075863.200386:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075863.200407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075863.200420:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff8b10037de300 x1855007612550016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075863.200434:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612550016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.200444:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075863.200448:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075863.200451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075863.200458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075863.200508:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075863.200523:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff8b10037ddf80 x1855007612550144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075863.200540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612550144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.200550:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075863.200554:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075863.200558:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075864.095767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075864.095997:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075865.119803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075865.119950:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075866.143815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075866.143876:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075867.167756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075867.167835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075868.191779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075868.191874:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075868.319793:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075868.319802:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075868.319805:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075868.319813:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2856 00000100:00080000:0.0:1769075868.319823:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075868.319829:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075868.319861:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612550272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075868.319885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075868.319888:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075868.319890:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075868.319893:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2856 00000100:00080000:0.0:1769075868.319895:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075868.319897:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075868.319904:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075868.319913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612550400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075868.319920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075868.319923:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075868.319925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075868.319927:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2856 00000100:00080000:0.0:1769075868.319930:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075868.319932:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075868.319935:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075868.319941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612550272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075868.319944:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612550528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075868.319953:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075868.319956:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075868.319957:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075868.319961:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2856 00000100:00080000:0.0:1769075868.319964:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075868.319968:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075868.319973:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075868.319986:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612550656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075868.320001:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075868.320033:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff8b1006025180 x1855007612550272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075868.320046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612550272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075868.320054:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075868.320057:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075868.320060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612550272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075868.320065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550400:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075868.320079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075868.320086:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff8b1006026680 x1855007612550400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075868.320094:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612550400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075868.320104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075868.320106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075868.320108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075868.320125:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075868.320136:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075868.320142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff8b1006027100 x1855007612550528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075868.320149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612550528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075868.320154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075868.320156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075868.320158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075868.320161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075868.320171:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075868.320177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff8b1006027800 x1855007612550656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075868.320185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612550656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075868.320189:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075868.320191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075868.320194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075869.215755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075869.215889:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075870.239727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075870.239830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075871.263779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075871.263868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075872.287749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075872.287904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075873.311869:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075873.311893:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075873.439860:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075873.439901:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075873.439912:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075873.439929:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2861 00000100:00080000:0.0:1769075873.439951:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075873.439964:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075873.440039:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612550784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075873.440079:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075873.440085:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075873.440088:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075873.440093:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2861 00000100:00080000:0.0:1769075873.440098:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075873.440102:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075873.440116:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075873.440135:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612550912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075873.440149:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075873.440155:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075873.440158:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075873.440162:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2861 00000100:00080000:0.0:1769075873.440166:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075873.440173:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075873.440181:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075873.440199:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612551040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075873.440201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612550784:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075873.440258:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075873.440264:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075873.440267:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075873.440273:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2861 00000100:00080000:0.0:1769075873.440277:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075873.440284:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075873.440336:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075873.440358:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612551168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075873.440369:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075873.440444:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff8b1006026a00 x1855007612550784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075873.440472:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612550784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075873.440487:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075873.440493:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075873.440498:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612550784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075873.440510:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075873.440539:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075873.440553:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff8b1006025880 x1855007612550912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075873.440569:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612550912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075873.440579:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075873.440583:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075873.440587:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612550912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075873.440619:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075873.440698:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075873.440715:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff8b1006024000 x1855007612551040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075873.440732:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612551040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075873.440742:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075873.440747:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075873.440750:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075873.440758:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075873.440781:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075873.440795:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff8b1006026300 x1855007612551168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075873.440820:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612551168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075873.440837:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075873.440843:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075873.440850:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075874.335747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075874.336004:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075875.359757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075875.359950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075876.383900:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075876.383971:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075877.407756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075877.407897:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075878.431768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075878.431859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075878.559875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075878.559888:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075878.559894:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075878.559905:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2866 00000100:00080000:1.0:1769075878.559919:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075878.559928:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075878.560002:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612551296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075878.560055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075878.560064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075878.560067:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075878.560075:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2866 00000100:00080000:1.0:1769075878.560093:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075878.560102:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075878.560119:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075878.560152:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612551424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075878.560170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612551296:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075878.560201:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075878.560248:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075878.560252:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075878.560277:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2866 00000100:00080000:1.0:1769075878.560286:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075878.560294:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075878.560307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075878.560318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075878.560338:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612551552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075878.560362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075878.560368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769075878.560374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff8b1006135880 x1855007612551296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075878.560377:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075878.560397:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2866 00000100:00080000:1.0:1769075878.560408:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075878.560416:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612551296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075878.560418:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075878.560433:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769075878.560443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.560453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075878.560483:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612551296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075878.560485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612551680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075878.560524:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075878.560574:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075878.560591:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff8b1006137b80 x1855007612551424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.560617:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612551424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.560636:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.560642:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075878.560649:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075878.560663:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075878.560758:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075878.560776:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff8b1006135c00 x1855007612551552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.560794:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612551552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.560805:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.560810:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075878.560814:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075878.560822:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075878.560864:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075878.560875:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff8b1006136300 x1855007612551680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.560898:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612551680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.560908:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.560911:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075878.560914:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075879.455788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075879.455972:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075880.479957:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075880.479968:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075881.503726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075881.503825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075882.527759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075882.527940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075883.551770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075883.551847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075883.679894:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.679908:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075883.679915:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.679928:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2871 00000100:00080000:3.0:1769075883.679942:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.680049:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075883.680098:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612551808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075883.680158:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.680164:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075883.680167:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.680172:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2871 00000100:00080000:3.0:1769075883.680177:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.680184:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075883.680195:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075883.680213:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612551936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075883.680231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612551808:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075883.680234:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.680248:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075883.680250:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.680256:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2871 00000100:00080000:3.0:1769075883.680262:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.680267:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075883.680274:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075883.680292:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612552064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075883.680314:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.680318:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075883.680320:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.680331:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2871 00000800:00000100:1.0:1769075883.680333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075883.680338:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.680346:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075883.680353:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075883.680369:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612552192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075883.680392:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff8b10037dc000 x1855007612551808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.680423:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612551808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.680437:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.680443:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075883.680449:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612551808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075883.680477:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551936:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075883.680504:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075883.680538:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff8b10037dea00 x1855007612551936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.680555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612551936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.680566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.680571:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075883.680575:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612551936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075883.680583:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075883.680606:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075883.680618:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff8b10037dd880 x1855007612552064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.680633:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612552064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.680675:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.680679:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075883.680683:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075883.680691:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075883.680715:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075883.680729:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff8b10037df100 x1855007612552192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.680743:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612552192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.680753:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.680757:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075883.680761:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075884.575777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075884.575858:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075885.599781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075885.599911:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075886.623782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075886.623858:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075887.647793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075887.647854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075888.671718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075888.671750:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075888.799748:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.799759:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075888.799764:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.799776:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2876 00000100:00080000:3.0:1769075888.799787:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.799795:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075888.799832:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612552320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.799867:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.799872:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075888.799875:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.799880:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2876 00000100:00080000:3.0:1769075888.799885:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.799890:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075888.799900:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075888.799918:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612552448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.799932:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.799936:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075888.799939:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.799945:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2876 00000100:00100000:0.0:1769075888.799950:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612552320:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075888.799952:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.799972:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075888.799979:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075888.799993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612552576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.800006:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.800010:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075888.800013:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.800017:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2876 00000800:00000100:0.0:1769075888.800022:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769075888.800024:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.800031:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075888.800057:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075888.800073:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612552704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075888.800078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff8b10037ddc00 x1855007612552320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075888.800103:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612552320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075888.800113:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075888.800193:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075888.800198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612552320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075888.800205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769075888.800237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075888.800254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff8b10037df800 x1855007612552448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075888.800270:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612552448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075888.800276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075888.800278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075888.800281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769075888.800300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075888.800326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075888.800333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff8b10037dc700 x1855007612552576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075888.800341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612552576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075888.800360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075888.800363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075888.800365:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075888.800369:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769075888.800382:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075888.800389:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff8b10037dce00 x1855007612552704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075888.800397:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612552704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075888.800402:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075888.800404:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769075888.800406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075889.695875:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075889.695915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075890.719762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075890.719885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075891.743783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075891.743877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075892.767774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075892.767907:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075893.791858:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075893.791937:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075893.919806:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.919823:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075893.919831:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.919872:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2882 00000100:00080000:0.0:1769075893.919884:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.919892:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075893.919944:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612552832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.920052:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.920058:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075893.920061:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.920067:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2882 00000100:00080000:0.0:1769075893.920073:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.920079:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075893.920090:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075893.920111:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612552960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.920127:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.920132:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075893.920135:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075893.920144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612552832:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075893.920146:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2882 00000100:00080000:0.0:1769075893.920158:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.920165:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075893.920172:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075893.920189:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612553088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.920203:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.920207:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075893.920209:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.920215:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2882 00000100:00080000:0.0:1769075893.920237:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.920241:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075893.920247:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075893.920259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769075893.920271:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612553216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075893.920334:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff8b1006026d80 x1855007612552832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.920382:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612552832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.920393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.920398:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075893.920403:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612552832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075893.920447:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075893.920483:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075893.920502:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff8b1006027b80 x1855007612552960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.920519:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612552960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.920530:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.920553:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075893.920557:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612552960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075893.920566:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075893.920594:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075893.920609:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff8b1006025f80 x1855007612553088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.920633:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612553088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.920703:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.920707:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075893.920712:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075893.920720:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075893.920748:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075893.920763:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff8b1006024a80 x1855007612553216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.920794:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612553216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.920805:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.920809:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075893.920813:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075894.815796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075894.815877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075895.839836:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075895.839874:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075896.863827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075896.863922:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075897.887929:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075897.887993:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075898.911733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075898.911782:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075899.039807:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.039816:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075899.039819:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.039827:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2887 00000100:00080000:3.0:1769075899.039839:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.039844:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075899.039897:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612553344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.039935:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.039938:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075899.039939:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.039942:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2887 00000100:00080000:3.0:1769075899.039944:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.039948:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075899.039955:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.039966:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612553472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.039979:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.039984:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769075899.039986:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612553344:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075899.039986:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.039993:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2887 00000100:00080000:3.0:1769075899.039998:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.040000:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075899.040003:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.040011:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612553600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075899.040017:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075899.040020:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075899.040021:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075899.040024:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2887 00000100:00080000:3.0:1769075899.040026:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075899.040028:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075899.040031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075899.040038:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612553728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075899.040090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075899.040114:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075899.040135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769075899.935717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075899.935771:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075900.959777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075900.959852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075901.983795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075901.983831:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000400:00000100:2.0:1769075902.111896:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075902.111937:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075902.111958:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075902.112002:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075902.112012:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075902.112019:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075902.112050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075899/real 1769075902] req@ffff8b10037dce00 x1855007612553472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075902.112073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612553472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075902.112083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075902.112088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075902.112092:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075902.112098:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075899/real 1769075902] req@ffff8b10037dc700 x1855007612553600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075902.112106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612553600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075902.112112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075902.112115:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075902.112117:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075902.112122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075899/real 1769075902] req@ffff8b10037df800 x1855007612553728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075902.112135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612553728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075902.112140:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075902.112142:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075902.112146:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075902.112150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075899/real 1769075902] req@ffff8b10037df480 x1855007612553344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075902.112160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612553344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075902.112165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075902.112167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075902.112169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612553344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075903.007810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075903.007853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075904.031941:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075904.031990:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075904.159851:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075904.159866:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075904.159873:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075904.159884:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2892 00000100:00080000:0.0:1769075904.159898:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075904.159906:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075904.159945:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612553856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075904.159990:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075904.159996:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075904.160000:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075904.160008:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2892 00000100:00080000:0.0:1769075904.160015:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075904.160022:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075904.160037:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075904.160060:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612553984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075904.160079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612553856:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075904.160082:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075904.160096:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075904.160100:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075904.160109:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2892 00000100:00080000:0.0:1769075904.160116:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075904.160123:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075904.160133:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075904.160156:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612554112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075904.160173:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075904.160179:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075904.160185:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769075904.160188:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075904.160197:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2892 00000100:00080000:0.0:1769075904.160209:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075904.160217:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075904.160225:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075904.160242:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612554240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075904.160259:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff8b1006024e00 x1855007612553856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.160308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612553856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.160327:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.160335:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075904.160342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612553856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075904.160379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075904.160423:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075904.160448:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff8b1006024380 x1855007612553984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.160472:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612553984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.160487:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.160492:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075904.160497:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612553984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075904.160508:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075904.160545:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075904.160566:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff8b1006027480 x1855007612554112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.160587:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612554112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.160600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.160606:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075904.160613:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075904.160623:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075904.161261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075904.161290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff8b1006024700 x1855007612554240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.161313:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612554240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.161351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.161356:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075904.161360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075905.055790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075905.055839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075906.079789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075906.079905:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075907.103777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075907.103819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075908.127727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075908.127778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075909.151883:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075909.151900:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769075909.279814:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.279825:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075909.279830:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.279841:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2897 00000100:00080000:3.0:1769075909.279853:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.279860:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075909.279896:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612554368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.279921:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.279924:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075909.279925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.279928:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2897 00000100:00080000:3.0:1769075909.279931:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.279934:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075909.279942:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075909.279951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612554496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.279960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.279964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769075909.279967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612554368:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075909.279967:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.279974:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2897 00000100:00080000:3.0:1769075909.279978:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.279980:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075909.279984:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075909.279993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612554624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.280000:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.280003:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075909.280005:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.280008:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2897 00000100:00080000:3.0:1769075909.280010:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.280012:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075909.280015:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075909.280023:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612554752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075909.280031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075909.280063:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff8b10037ddc00 x1855007612554368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.280077:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612554368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.280085:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.280088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075909.280091:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612554368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075909.280110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554496:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075909.280125:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075909.280133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff8b10037df100 x1855007612554496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.280142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612554496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.280152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.280155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075909.280157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075909.280161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075909.280172:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075909.280178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff8b10037dd880 x1855007612554624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.280185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612554624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.280191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.280193:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075909.280195:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075909.280198:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075909.280209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075909.280215:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff8b10037dea00 x1855007612554752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.280223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612554752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.280228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.280230:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075909.280233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612554752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075910.175832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075910.175996:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075911.199819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075911.199895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075912.223771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075912.223854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075913.247697:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075913.247775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075914.271756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075914.271828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075914.399707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.399720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075914.399725:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.399735:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2902 00000100:00080000:3.0:1769075914.399748:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.399756:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075914.399811:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612554880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.399840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.399843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075914.399844:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.399847:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2902 00000100:00080000:3.0:1769075914.399850:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.399852:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075914.399863:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075914.399872:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612555008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.399880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.399883:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075914.399884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.399887:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2902 00000100:00080000:3.0:1769075914.399889:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.399892:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075914.399896:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075914.399903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612554880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075914.399905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612555136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.399917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.399920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075914.399921:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.399924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2902 00000100:00080000:3.0:1769075914.399926:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.399929:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075914.399932:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075914.399939:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612555264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075914.399994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075914.400031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075914] req@ffff8b10037dc000 x1855007612554880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075914.400047:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612554880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075914.400057:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075914.400061:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075914.400064:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612554880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075914.400070:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075914.400085:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075914.400094:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075914] req@ffff8b10037ddf80 x1855007612555008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075914.400102:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612555008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075914.400108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075914.400110:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075914.400112:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555008:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075914.400133:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075914.400148:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075914.400154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075914] req@ffff8b10037de300 x1855007612555136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075914.400163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612555136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075914.400168:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075914.400170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075914.400172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075914.400176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075914.400186:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075914.400193:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075914] req@ffff8b10037dd180 x1855007612555264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075914.400206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612555264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075914.400215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075914.400219:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075914.400222:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075915.295712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075915.295769:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075916.319757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075916.319793:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075917.343767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075917.343872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075918.367701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075918.367808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075919.391741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075919.391780:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075919.519714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.519726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075919.519733:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.519743:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2907 00000100:00080000:3.0:1769075919.519764:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.519771:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075919.519814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612555392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.519845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.519848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075919.519849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.519853:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2907 00000100:00080000:3.0:1769075919.519855:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.519858:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075919.519867:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075919.519877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612555520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.519885:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.519888:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075919.519889:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.519892:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2907 00000100:00080000:3.0:1769075919.519894:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075919.519898:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612555392:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075919.519899:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075919.519910:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075919.519920:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612555648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.519927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.519930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075919.519931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.519934:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2907 00000100:00080000:3.0:1769075919.519936:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.519939:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075919.519943:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075919.519951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612555776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075919.519977:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075919.520015:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff8b10037dd500 x1855007612555392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.520035:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612555392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.520044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.520048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075919.520051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612555392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075919.520073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075919.520088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075919.520097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff8b10037ded80 x1855007612555520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.520106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612555520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.520111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.520113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075919.520115:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075919.520119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075919.520131:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075919.520137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff8b10037de680 x1855007612555648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.520145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612555648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.520155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.520157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075919.520159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075919.520162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075919.520174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075919.520180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff8b10037dc380 x1855007612555776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.520187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612555776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.520192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.520194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075919.520197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612555776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075920.415735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075920.415899:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075921.439729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075921.439795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075922.463693:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075922.463795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075923.487837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075923.487865:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075924.511747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075924.511829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075924.639775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075924.639786:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075924.639791:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075924.639803:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2912 00000100:00080000:0.0:1769075924.639814:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075924.639820:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075924.639853:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612555904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075924.639879:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075924.639882:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075924.639884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075924.639887:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2912 00000100:00080000:0.0:1769075924.639890:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075924.639893:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075924.639901:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075924.639915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612556032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075924.639918:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612555904:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075924.639929:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075924.639934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075924.639935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075924.639939:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2912 00000100:00080000:0.0:1769075924.639943:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075924.639945:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075924.639949:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075924.639959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612556160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075924.639966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075924.639969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075924.639970:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075924.639972:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2912 00000100:00080000:0.0:1769075924.639975:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075924.639978:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075924.639981:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075924.640001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612556288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075924.640002:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075924.640040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff8b1006024700 x1855007612555904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.640055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612555904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.640063:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.640067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075924.640070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612555904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075924.640092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556032:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075924.640189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075924.640214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff8b1006027480 x1855007612556032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.640224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612556032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.640231:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.640233:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075924.640235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075924.640240:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075924.640257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075924.640265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff8b1006024380 x1855007612556160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.640273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612556160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.640278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.640281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075924.640283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075924.640286:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075924.640297:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075924.640304:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff8b1006024e00 x1855007612556288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.640311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612556288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.640317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.640318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075924.640320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075925.535835:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075925.535962:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075926.559758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075926.559866:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075927.583760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075927.583904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075928.607809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075928.607821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075929.631750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075929.631824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075929.759741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.759753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075929.759757:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.759767:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2917 00000100:00080000:0.0:1769075929.759779:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.759786:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075929.759825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612556416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075929.759852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.759856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075929.759859:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.759863:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2917 00000100:00080000:0.0:1769075929.759867:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.759872:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075929.759882:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075929.759889:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612556416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075929.759897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612556544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075929.759911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.759915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075929.759917:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.759922:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2917 00000100:00080000:0.0:1769075929.759926:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.759930:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075929.759935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075929.759947:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612556672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075929.759949:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075929.759961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.759965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075929.759967:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.759972:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2917 00000100:00080000:0.0:1769075929.759976:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.759980:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075929.759984:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff8b1006024a80 x1855007612556416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769075929.759988:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075929.760005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612556800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075929.760006:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612556416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.760035:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.760040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075929.760044:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612556416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075929.760063:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075929.760078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075929.760087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff8b1006025f80 x1855007612556544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.760100:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612556544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.760106:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.760108:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075929.760111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075929.760115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075929.760126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075929.760133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff8b1006027b80 x1855007612556672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.760140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612556672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.760146:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.760147:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075929.760149:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075929.760153:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075929.760163:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075929.760179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff8b1006026d80 x1855007612556800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.760192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612556800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.760206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.760210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075929.760214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612556800:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769075930.655777:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075930.655778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075931.679770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075931.679871:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075932.703748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075932.703806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075933.727724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075933.727800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075934.751796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075934.751900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075934.879890:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075934.879906:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075934.879912:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075934.879926:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2922 00000100:00080000:1.0:1769075934.879940:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075934.879950:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075934.879997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612556928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075934.880035:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075934.880041:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075934.880044:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075934.880049:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2922 00000100:00080000:1.0:1769075934.880054:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075934.880058:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075934.880069:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075934.880085:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612557056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075934.880099:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075934.880104:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075934.880106:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075934.880111:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2922 00000100:00080000:1.0:1769075934.880115:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075934.880119:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075934.880127:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075934.880139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612556928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075934.880145:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612557184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075934.880168:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075934.880172:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075934.880174:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075934.880179:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2922 00000100:00080000:1.0:1769075934.880182:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075934.880187:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075934.880194:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075934.880211:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612557312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075934.880319:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075934.880384:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff8b1006137800 x1855007612556928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.880424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612556928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.880446:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.880455:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075934.880464:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612556928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075934.880481:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557056:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075934.880534:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075934.880565:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff8b1006135180 x1855007612557056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.880594:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612557056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.880613:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.880621:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075934.880628:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075934.880726:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075934.880777:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075934.880805:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff8b1006136a00 x1855007612557184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.880833:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612557184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.880852:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.880859:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075934.880867:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075934.880880:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075934.880928:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075934.880956:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff8b1006136300 x1855007612557312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.880983:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612557312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.881000:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.881007:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075934.881014:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075935.775764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075935.775860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075936.799736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075936.799830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075937.823771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075937.823834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075938.847704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075938.847767:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075939.871707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075939.871795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075939.999774:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.999786:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075939.999790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.999800:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2928 00000100:00080000:1.0:1769075939.999813:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.999820:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075939.999872:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612557440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.999895:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.999898:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075939.999900:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.999903:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2928 00000100:00080000:1.0:1769075939.999906:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.999908:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075939.999916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075939.999924:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612557568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.999938:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.999941:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075939.999942:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.999945:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2928 00000100:00080000:1.0:1769075939.999948:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.999950:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075939.999954:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075939.999962:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612557696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.999969:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.999971:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075939.999972:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.999974:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2928 00000100:00080000:1.0:1769075939.999976:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.999979:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075939.999981:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075939.999988:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612557824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075939.999997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612557440:192.168.204.102@tcp:250:kworker.0 00000800:00000100:3.0:1769075940.000087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075940.000141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075940] req@ffff8b1006135c00 x1855007612557440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075940.000156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612557440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075940.000164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075940.000168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075940.000170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612557440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075940.000193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075940.000212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075940.000222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075940] req@ffff8b1006137b80 x1855007612557568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075940.000231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612557568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075940.000237:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075940.000239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075940.000242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075940.000246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075940.000258:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075940.000265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075940] req@ffff8b1006135880 x1855007612557696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075940.000273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612557696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075940.000279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075940.000281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075940.000283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075940.000302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075940.000313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075940.000319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075940] req@ffff8b1006135500 x1855007612557824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075940.000327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612557824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075940.000333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075940.000334:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075940.000336:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612557824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075940.895773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075940.895854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075941.919726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075941.919788:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075942.943740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075942.943827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075943.967792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075943.967891:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075944.991864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075944.991984:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075945.119798:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075945.119811:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075945.119817:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075945.119829:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2933 00000100:00080000:1.0:1769075945.119843:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075945.119852:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075945.119900:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612557952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075945.119947:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075945.119959:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075945.119962:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075945.119970:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2933 00000100:00080000:1.0:1769075945.119978:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075945.119986:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075945.120003:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075945.120042:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612558080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075945.120065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612557952:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075945.120068:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075945.120078:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075945.120084:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075945.120135:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2933 00000100:00080000:1.0:1769075945.120142:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075945.120148:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075945.120158:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075945.120177:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612558208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075945.120191:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075945.120196:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075945.120199:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075945.120203:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2933 00000100:00080000:1.0:1769075945.120208:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075945.120213:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075945.120221:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075945.120235:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612558336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075945.120248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075945.120317:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff8b1006137100 x1855007612557952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.120351:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612557952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.120374:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.120383:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075945.120391:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612557952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075945.120431:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075945.120480:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075945.120506:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff8b1006134700 x1855007612558080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.120526:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612558080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.120537:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.120542:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075945.120546:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075945.120555:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075945.120583:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075945.120597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff8b1006136d80 x1855007612558208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.120612:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612558208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.120623:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.120626:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075945.120630:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075945.120680:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075945.120707:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075945.120738:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff8b1006136680 x1855007612558336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.120754:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612558336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.120765:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.120769:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075945.120773:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075946.015799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075946.015894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075947.039728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075947.039851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075948.063753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075948.063832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075949.087780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075949.087885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075950.111819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075950.111934:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075950.239929:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075950.239943:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075950.239960:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075950.239973:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2938 00000100:00080000:0.0:1769075950.239987:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075950.239996:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075950.240089:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612558464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075950.240138:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075950.240143:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075950.240146:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075950.240151:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2938 00000100:00080000:0.0:1769075950.240157:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075950.240162:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075950.240176:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075950.240192:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612558592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075950.240207:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075950.240211:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075950.240215:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075950.240222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612558464:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075950.240223:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2938 00000100:00080000:0.0:1769075950.240231:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075950.240245:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075950.240251:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075950.240272:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612558720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075950.240286:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075950.240290:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075950.240292:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075950.240297:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2938 00000100:00080000:0.0:1769075950.240306:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075950.240311:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075950.240317:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075950.240328:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612558848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075950.240350:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075950.240409:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff8b1006026300 x1855007612558464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.240428:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612558464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.240438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.240442:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075950.240446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612558464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075950.240473:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558592:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075950.240494:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075950.240504:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff8b1006024000 x1855007612558592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.240513:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612558592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.240519:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.240522:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075950.240524:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075950.240529:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075950.240542:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075950.240549:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff8b1006025880 x1855007612558720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.240557:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612558720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.240562:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.240564:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075950.240566:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075950.240571:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075950.240583:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075950.240590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff8b1006026a00 x1855007612558848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.240610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612558848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.240620:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.240623:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075950.240627:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612558848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075951.135778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075951.135938:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075952.159791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075952.159869:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075953.183772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075953.183849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075954.207781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075954.207912:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075955.231788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075955.231864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075955.359878:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075955.359890:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075955.359894:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075955.359905:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2943 00000100:00080000:3.0:1769075955.359979:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075955.359989:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075955.360031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612558976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075955.360076:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075955.360093:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075955.360097:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075955.360104:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2943 00000100:00080000:3.0:1769075955.360111:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075955.360119:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075955.360134:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075955.360187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612558976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075955.360216:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612559104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075955.360236:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075955.360242:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075955.360246:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075955.360254:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2943 00000100:00080000:3.0:1769075955.360270:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075955.360276:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075955.360286:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075955.360297:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075955.360303:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612559232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075955.360318:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075955.360324:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075955.360327:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075955.360336:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2943 00000100:00080000:3.0:1769075955.360351:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075955.360358:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075955.360365:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff8b10037dca80 x1855007612558976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075955.360367:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075955.360384:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612559360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075955.360422:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612558976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075955.360438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075955.360445:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075955.360451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612558976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075955.360502:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075955.360546:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075955.360564:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff8b10037dc380 x1855007612559104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075955.360580:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612559104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075955.360590:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075955.360594:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075955.360598:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075955.360606:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075955.360627:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075955.360638:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff8b10037de680 x1855007612559232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075955.360668:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612559232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075955.360678:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075955.360682:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075955.360731:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075955.360740:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075955.360769:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075955.360785:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff8b10037ded80 x1855007612559360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075955.360801:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612559360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075955.360827:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075955.360831:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075955.360834:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075956.255791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075956.255934:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075957.279804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075957.279993:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075958.303759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075958.303915:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075959.327793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075959.327805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075960.351882:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075960.351953:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075960.479855:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075960.479871:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075960.479879:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075960.479895:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2948 00000100:00080000:0.0:1769075960.479913:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075960.479926:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075960.479980:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612559488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075960.480029:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075960.480038:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075960.480043:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075960.480054:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2948 00000100:00080000:0.0:1769075960.480062:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075960.480071:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075960.480089:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075960.480091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612559488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075960.480129:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612559616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075960.480158:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075960.480166:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075960.480171:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075960.480180:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2948 00000100:00080000:0.0:1769075960.480189:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075960.480199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769075960.480204:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075960.480216:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075960.480237:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612559744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075960.480251:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075960.480256:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075960.480259:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075960.480264:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2948 00000100:00080000:0.0:1769075960.480270:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075960.480273:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075960.480279:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075960.480289:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612559872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075960.480366:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff8b1006027800 x1855007612559488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075960.480395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612559488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075960.480409:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075960.480415:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075960.480593:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612559488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075960.480759:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075960.480810:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075960.480862:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff8b1006027100 x1855007612559616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075960.480893:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612559616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075960.480913:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075960.480920:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075960.480928:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075960.480942:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075960.480976:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075960.480993:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff8b1006026680 x1855007612559744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075960.481019:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612559744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075960.481030:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075960.481034:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075960.481039:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075960.481047:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075960.481068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075960.481081:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff8b1006025180 x1855007612559872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075960.481096:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612559872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075960.481106:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075960.481110:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075960.481114:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612559872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075961.375766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075961.375998:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075962.399847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075962.399903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075963.423816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075963.423911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075964.447822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075964.447974:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075965.471718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075965.471856:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075965.599877:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075965.599897:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075965.599903:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075965.599915:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2953 00000100:00080000:3.0:1769075965.599930:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075965.599938:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769075965.599987:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612560000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075965.600033:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075965.600041:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075965.600045:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075965.600053:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2953 00000100:00080000:3.0:1769075965.600060:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075965.600067:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075965.600082:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075965.600108:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612560128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075965.600130:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075965.600137:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075965.600141:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075965.600144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612560000:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769075965.600148:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2953 00000100:00080000:3.0:1769075965.600155:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075965.600162:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075965.600171:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075965.600188:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612560256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075965.600204:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075965.600210:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075965.600213:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075965.600219:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2953 00000100:00080000:3.0:1769075965.600224:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075965.600230:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769075965.600238:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075965.600256:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612560384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075965.600298:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075965.600367:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075965.600412:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075966.495878:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075966.495902:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075967.519823:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075967.519906:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075968.543822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075968.543923:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075968.671963:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769075968.672008:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075968.672025:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075968.672086:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075968.672094:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769075968.672102:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769075968.672182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075968] req@ffff8b10037dd180 x1855007612560128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075968.672221:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612560128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075968.672237:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075968.672244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075968.672251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075968.672262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075968] req@ffff8b10037de300 x1855007612560256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075968.672280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612560256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075968.672292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075968.672297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075968.672303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075968.672311:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075968] req@ffff8b10037ddf80 x1855007612560384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075968.672329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612560384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075968.672341:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075968.672347:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075968.672352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075968.672361:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075968] req@ffff8b10037dd500 x1855007612560000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075968.672389:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612560000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075968.672400:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075968.672404:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075968.672409:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612560000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769075969.567807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075969.567922:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075970.591776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075970.591882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075970.719863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075970.719891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075970.719899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075970.719914:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2958 00000100:00080000:1.0:1769075970.719932:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075970.719943:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075970.720003:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612560512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075970.720069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075970.720076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075970.720079:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075970.720087:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2958 00000100:00080000:1.0:1769075970.720093:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075970.720099:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075970.720117:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075970.720128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612560512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075970.720151:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612560640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075970.720171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075970.720176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075970.720179:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075970.720184:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2958 00000100:00080000:1.0:1769075970.720190:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075970.720196:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075970.720206:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075970.720234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612560768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075970.720252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769075970.720255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075970.720263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075970.720266:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075970.720274:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2958 00000100:00080000:1.0:1769075970.720290:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075970.720296:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075970.720305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075970.720318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff8b1006137480 x1855007612560512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075970.720329:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612560896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075970.720362:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612560512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075970.720396:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075970.720404:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075970.720411:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612560512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075970.720448:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075970.720479:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075970.720497:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff8b1006134a80 x1855007612560640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075970.720526:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612560640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075970.720543:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075970.720549:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075970.720555:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075970.720567:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075970.720611:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075970.720636:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff8b1006134e00 x1855007612560768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075970.720734:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612560768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075970.720749:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075970.720755:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075970.720760:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075970.720772:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075970.720814:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075970.720835:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff8b1006136680 x1855007612560896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075970.720858:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612560896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075970.720900:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075970.721062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075970.721072:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612560896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075971.615814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075971.615913:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075972.639764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075972.639901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075973.663884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075973.663890:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075974.687791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075974.687935:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075975.711795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075975.711914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075975.839768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075975.839782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075975.839789:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075975.839801:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2963 00000100:00080000:1.0:1769075975.839815:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075975.839824:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075975.839870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612561024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075975.839919:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075975.839924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075975.839927:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075975.839932:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2963 00000100:00080000:1.0:1769075975.839937:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075975.839941:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075975.839953:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075975.839969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612561152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075975.839984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075975.839990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075975.839993:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075975.839997:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2963 00000100:00080000:1.0:1769075975.840003:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075975.840008:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075975.840015:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075975.840023:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612561024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075975.840035:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612561280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075975.840060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075975.840064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075975.840066:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075975.840071:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2963 00000100:00080000:1.0:1769075975.840077:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075975.840081:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075975.840087:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075975.840104:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612561408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075975.840122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075975.840188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff8b1006136d80 x1855007612561024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075975.840215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612561024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075975.840229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075975.840235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075975.840240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612561024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075975.840250:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561152:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075975.840277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075975.840293:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff8b1006134700 x1855007612561152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075975.840309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612561152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075975.840327:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075975.840332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075975.840336:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075975.840365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075975.840387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075975.840399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff8b1006137100 x1855007612561280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075975.840414:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612561280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075975.840424:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075975.840428:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075975.840431:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075975.840438:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075975.840458:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075975.840470:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff8b1006135500 x1855007612561408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075975.840484:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612561408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075975.840494:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075975.840498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075975.840502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075976.735760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075976.735872:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075977.759845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075977.759893:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075978.783784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075978.783904:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075979.807765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075979.807812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075980.831872:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075980.831902:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075980.959746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075980.959764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075980.959771:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075980.959788:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2969 00000100:00080000:1.0:1769075980.959807:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075980.959819:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769075980.959883:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612561536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075980.959927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075980.959932:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075980.959935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075980.959941:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2969 00000100:00080000:1.0:1769075980.959945:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075980.959950:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075980.959968:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075980.959994:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612561664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075980.960018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075980.960025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075980.960030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075980.960039:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2969 00000100:00080000:1.0:1769075980.960047:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075980.960057:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612561536:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769075980.960065:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075980.960082:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075980.960109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612561792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075980.960126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075980.960131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075980.960133:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075980.960139:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2969 00000100:00080000:1.0:1769075980.960145:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075980.960150:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769075980.960158:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075980.960193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075980.960201:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612561920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075980.960278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075980] req@ffff8b1006135880 x1855007612561536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075980.960309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612561536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075980.960325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075980.960332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075980.960337:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612561536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075980.960349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561664:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075980.960385:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075980.960402:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075980] req@ffff8b1006137b80 x1855007612561664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075980.960418:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612561664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075980.960429:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075980.960432:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075980.960437:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075980.960474:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075980.960539:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075980.960557:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075980] req@ffff8b1006135c00 x1855007612561792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075980.960574:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612561792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075980.960585:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075980.960590:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075980.960594:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075980.960602:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075980.960625:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075980.960637:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075980] req@ffff8b1006136300 x1855007612561920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075980.960692:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612561920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075980.960703:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075980.960707:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075980.960711:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612561920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075981.855807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075981.855897:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075982.879726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075982.879806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075983.903826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075983.903960:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075984.927751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075984.927822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075985.951803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075985.952023:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075986.079757:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075986.079767:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075986.079772:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075986.079780:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2974 00000100:00080000:0.0:1769075986.079791:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075986.079797:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075986.079833:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612562048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075986.079858:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075986.079861:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075986.079863:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075986.079866:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2974 00000100:00080000:0.0:1769075986.079869:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075986.079872:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075986.079881:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075986.079892:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612562176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075986.079903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612562048:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075986.079904:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075986.079913:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075986.079916:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075986.079920:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2974 00000100:00080000:0.0:1769075986.079923:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075986.079927:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075986.079931:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075986.079941:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612562304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075986.079948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075986.079952:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075986.079953:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075986.079956:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2974 00000100:00080000:0.0:1769075986.079958:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075986.079961:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075986.079965:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075986.079974:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612562432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075986.079979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075986.080028:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff8b1006025180 x1855007612562048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075986.080045:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612562048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075986.080054:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075986.080058:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075986.080062:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612562048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075986.080082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562176:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075986.080102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075986.080111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff8b1006026680 x1855007612562176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075986.080121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612562176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075986.080128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075986.080131:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075986.080133:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562176:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075986.080138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075986.080151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075986.080158:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff8b1006027100 x1855007612562304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075986.080173:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612562304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075986.080179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075986.080181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075986.080183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075986.080187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075986.080200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075986.080207:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff8b1006027800 x1855007612562432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075986.080216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612562432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075986.080222:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075986.080224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075986.080226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075986.975717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075986.975829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075987.999775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075987.999934:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075989.023778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075989.023948:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075990.047802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075990.047952:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075991.071729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075991.071814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075991.199726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.199734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075991.199738:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.199746:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2979 00000100:00080000:0.0:1769075991.199756:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.199761:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075991.199793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612562560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075991.199816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.199820:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075991.199821:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.199824:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2979 00000100:00080000:0.0:1769075991.199826:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.199829:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075991.199836:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075991.199844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612562688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075991.199850:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.199853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075991.199854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.199856:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2979 00000100:00080000:0.0:1769075991.199858:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.199860:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075991.199863:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075991.199868:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612562816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075991.199875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.199877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075991.199879:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.199882:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2979 00000100:00080000:0.0:1769075991.199884:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.199886:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075991.199890:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075991.199898:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612562560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769075991.199899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612562944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075991.199970:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075991.199999:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff8b1006026a00 x1855007612562560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075991.200020:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612562560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075991.200028:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075991.200031:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075991.200034:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612562560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769075991.200039:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562688:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769075991.200053:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075991.200061:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff8b1006025880 x1855007612562688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075991.200070:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612562688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075991.200075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075991.200077:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075991.200079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769075991.200083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075991.200094:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075991.200100:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff8b1006024000 x1855007612562816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075991.200108:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612562816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075991.200118:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075991.200120:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075991.200122:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769075991.200140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769075991.200151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769075991.200157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff8b1006026300 x1855007612562944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075991.200164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612562944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075991.200169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075991.200171:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769075991.200173:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612562944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075992.095730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075992.095823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075993.119792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075993.119981:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075994.143783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075994.143970:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075995.167855:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075995.167864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075996.191744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075996.191890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075996.319808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.319824:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075996.319830:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.319844:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2984 00000100:00080000:0.0:1769075996.319858:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.319886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769075996.319926:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612563072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.319965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.319969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075996.319971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.319977:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2984 00000100:00080000:0.0:1769075996.319982:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.319988:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075996.319999:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075996.320024:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612563200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.320039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.320044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075996.320046:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.320054:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2984 00000100:00080000:0.0:1769075996.320059:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.320064:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075996.320070:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075996.320086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612563328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075996.320093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612563072:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769075996.320112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.320117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075996.320119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.320125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2984 00000100:00080000:0.0:1769075996.320130:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.320134:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769075996.320140:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075996.320161:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612563456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075996.320267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075996.320343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff8b1006026d80 x1855007612563072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075996.320382:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612563072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075996.320405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075996.320416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075996.320428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612563072:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769075996.320442:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563200:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769075996.320480:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075996.320499:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff8b1006027b80 x1855007612563200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075996.320533:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612563200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075996.320542:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075996.320547:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075996.320552:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563200:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769075996.320581:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075996.320607:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075996.320622:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff8b1006025f80 x1855007612563328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075996.320639:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612563328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075996.320657:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075996.320662:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075996.320718:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769075996.320753:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769075996.320795:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769075996.320825:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff8b1006024a80 x1855007612563456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075996.320853:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612563456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075996.320881:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075996.320885:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769075996.320891:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769075997.215737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075997.215877:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075998.239807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075998.239942:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075999.263734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075999.263840:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076000.287719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076000.287789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076001.311767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076001.311871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076001.439830:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076001.439847:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076001.439854:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076001.439868:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2989 00000100:00080000:3.0:1769076001.439887:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076001.439898:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076001.439947:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612563584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076001.439985:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076001.439990:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076001.439992:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076001.439997:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2989 00000100:00080000:3.0:1769076001.440002:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076001.440006:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076001.440017:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076001.440032:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612563712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076001.440046:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076001.440050:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076001.440053:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076001.440058:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2989 00000100:00080000:3.0:1769076001.440064:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076001.440067:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612563584:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076001.440072:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076001.440087:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076001.440103:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612563840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076001.440116:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076001.440121:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076001.440125:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076001.440131:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2989 00000100:00080000:3.0:1769076001.440136:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076001.440140:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076001.440173:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076001.440218:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612563968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076001.440219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076001.440298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff8b10037dc000 x1855007612563584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076001.440323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612563584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076001.440336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076001.440342:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076001.440347:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612563584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076001.440358:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563712:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076001.440387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076001.440405:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff8b10037dea00 x1855007612563712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076001.440420:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612563712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076001.440430:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076001.440434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076001.440438:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563712:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076001.440466:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076001.440505:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076001.440516:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff8b10037dd880 x1855007612563840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076001.440529:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612563840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076001.440538:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076001.440542:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076001.440545:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076001.440551:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076001.440567:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076001.440578:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff8b10037df100 x1855007612563968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076001.440591:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612563968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076001.440600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076001.440603:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076001.440606:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612563968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076002.335857:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076002.335938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076003.359760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076003.359866:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076004.383788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076004.383880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076005.407776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076005.407898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076006.431760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076006.431883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076006.559784:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.559810:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076006.559815:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.559823:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2994 00000100:00080000:2.0:1769076006.559834:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.559850:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076006.559883:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612564096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.559906:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.559909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076006.559911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.559914:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2994 00000100:00080000:2.0:1769076006.559917:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.559919:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076006.559926:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076006.559933:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612564224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.559941:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.559944:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076006.559946:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.559949:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2994 00000100:00080000:2.0:1769076006.559951:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.559953:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076006.559958:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076006.559982:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612564352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.559991:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.559994:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076006.559995:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.559998:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2994 00000100:00080000:2.0:1769076006.560002:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.560007:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076006.560011:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076006.560022:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612564480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076006.560026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612564096:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769076006.560174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076006.560221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff8b1005d5b480 x1855007612564096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076006.560244:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612564096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076006.560257:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076006.560262:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076006.560267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612564096:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076006.560275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564224:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076006.560298:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076006.560309:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff8b1005d5b800 x1855007612564224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076006.560321:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612564224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076006.560336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076006.560340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076006.560343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564224:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076006.560349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076006.560365:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076006.560374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff8b1005d58e00 x1855007612564352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076006.560385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612564352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076006.560393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076006.560396:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076006.560399:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076006.560431:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076006.560463:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076006.560475:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff8b1005d5bb80 x1855007612564480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076006.560487:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612564480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076006.560495:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076006.560498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076006.560502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076007.455837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076007.455949:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076008.479796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076008.479854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076009.503845:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076009.503879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076010.527770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076010.527903:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076011.551725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076011.551806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076011.679767:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076011.679793:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076011.679799:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076011.679814:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 2999 00000100:00080000:0.0:1769076011.679830:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076011.679841:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076011.679900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612564608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076011.679933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076011.679938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076011.679940:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076011.679944:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2999 00000100:00080000:0.0:1769076011.679948:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076011.679951:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076011.679964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076011.680000:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612564736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076011.680048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076011.680049:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612564608:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076011.680060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076011.680068:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076011.680097:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2999 00000100:00080000:0.0:1769076011.680103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076011.680110:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076011.680119:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076011.680145:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612564864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076011.680159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076011.680165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076011.680168:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076011.680174:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 2999 00000100:00080000:0.0:1769076011.680180:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076011.680184:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076011.680192:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076011.680196:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076011.680215:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612564992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076011.680267:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff8b1006024e00 x1855007612564608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076011.680298:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612564608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076011.680314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076011.680341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076011.680348:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612564608:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076011.680384:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564736:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076011.680423:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076011.680444:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff8b1006024380 x1855007612564736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076011.680458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612564736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076011.680477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076011.680481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076011.680485:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564736:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076011.680491:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076011.680530:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076011.680543:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff8b1006027480 x1855007612564864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076011.680556:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612564864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076011.680565:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076011.680568:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076011.680571:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076011.680576:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076011.680594:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076011.680602:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff8b1006024700 x1855007612564992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076011.680614:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612564992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076011.680622:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076011.680625:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076011.680627:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612564992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076012.575783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076012.575983:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076013.599794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076013.599897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076014.623849:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076014.624067:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076015.647776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076015.647959:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076016.671875:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076016.671976:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076016.799783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.799811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076016.799819:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.799836:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3004 00000100:00080000:0.0:1769076016.799853:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.799866:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076016.799961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612565120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.800002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.800008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076016.800011:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.800017:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3004 00000100:00080000:0.0:1769076016.800023:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.800028:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076016.800040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076016.800058:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612565248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.800070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.800075:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076016.800077:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.800081:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3004 00000100:00080000:0.0:1769076016.800085:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.800089:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076016.800095:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076016.800109:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612565376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.800121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.800125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076016.800127:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.800132:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3004 00000100:00080000:0.0:1769076016.800136:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.800145:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076016.800150:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612565120:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076016.800152:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076016.800170:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612565504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076016.800248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076016.800294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff8b1006024700 x1855007612565120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076016.800319:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612565120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076016.800334:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076016.800339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076016.800345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612565120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076016.800355:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565248:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076016.800380:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076016.800394:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff8b1006027480 x1855007612565248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076016.800410:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612565248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076016.800421:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076016.800424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076016.800428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076016.800435:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076016.800456:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076016.800468:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff8b1006024380 x1855007612565376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076016.800482:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612565376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076016.800492:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076016.800496:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076016.800499:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076016.800532:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076016.800553:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076016.800565:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff8b1006024e00 x1855007612565504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076016.800580:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612565504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076016.800590:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076016.800594:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076016.800597:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076017.695782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076017.695866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076018.719774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076018.719842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076019.743767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076019.743803:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076020.767783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076020.767892:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076021.791765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076021.791887:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076021.919798:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076021.919835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076021.919842:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076021.919857:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3010 00000100:00080000:3.0:1769076021.919876:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076021.919887:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076021.919951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612565632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076021.919993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076021.919999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076021.920002:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076021.920008:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3010 00000100:00080000:3.0:1769076021.920012:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076021.920017:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076021.920032:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076021.920056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612565760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076021.920071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076021.920077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076021.920079:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076021.920085:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3010 00000100:00080000:3.0:1769076021.920089:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076021.920102:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076021.920105:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612565632:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076021.920112:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076021.920140:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612565888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076021.920159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076021.920164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076021.920167:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076021.920174:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3010 00000100:00080000:3.0:1769076021.920179:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076021.920189:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076021.920200:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076021.920224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076021.920238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612566016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076021.920294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff8b10037ddc00 x1855007612565632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076021.920326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612565632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076021.920342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076021.920348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076021.920354:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612565632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076021.920365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565760:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076021.920393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076021.920407:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff8b10037df480 x1855007612565760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076021.920424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612565760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076021.920435:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076021.920439:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076021.920443:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565760:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076021.920475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076021.920497:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076021.920508:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff8b10037df800 x1855007612565888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076021.920523:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612565888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076021.920534:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076021.920537:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076021.920541:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612565888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076021.920548:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076021.920567:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076021.920579:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff8b10037dc700 x1855007612566016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076021.920593:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612566016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076021.920612:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076021.920616:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076021.920619:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076022.815844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076022.815966:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076023.839901:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076023.839920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076024.863736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076024.863864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076025.887793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076025.887943:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076026.911805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076026.911936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076027.039827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076027.039840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076027.039845:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076027.039856:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3015 00000100:00080000:1.0:1769076027.039871:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076027.039880:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076027.039923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612566144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076027.039959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076027.039964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076027.039966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076027.039971:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3015 00000100:00080000:1.0:1769076027.039976:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076027.039982:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076027.039994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076027.040012:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612566272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076027.040020:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612566144:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076027.040052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076027.040060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076027.040065:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076027.040090:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3015 00000100:00080000:1.0:1769076027.040099:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076027.040108:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076027.040120:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076027.040130:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076027.040152:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612566400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076027.040167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076027.040171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076027.040175:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076027.040198:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3015 00000100:00080000:1.0:1769076027.040205:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076027.040217:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076027/real 1769076027] req@ffff8b1006136a00 x1855007612566144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076027.040228:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076027.040239:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076027.040267:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612566528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076027.040278:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612566144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076027.040304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076027.040311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076027.040319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612566144:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076027.040351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566272:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076027.040406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076027.040422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076027/real 1769076027] req@ffff8b1006135180 x1855007612566272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076027.040455:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612566272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076027.040472:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076027.040476:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076027.040480:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566272:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076027.040488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076027.040512:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076027.040525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076027/real 1769076027] req@ffff8b1006137800 x1855007612566400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076027.040539:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612566400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076027.040548:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076027.040551:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076027.040555:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076027.040562:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076027.040601:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076027.040614:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076027/real 1769076027] req@ffff8b1006136300 x1855007612566528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076027.040628:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612566528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076027.040638:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076027.040641:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076027.040646:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076027.935880:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076027.935984:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076028.959827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076028.959957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076029.983796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076029.983918:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076031.007841:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076031.007933:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076032.031780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076032.031914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076032.159816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076032.159834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076032.159841:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076032.159859:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3020 00000100:00080000:3.0:1769076032.159876:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076032.159886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076032.159946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612566656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076032.159984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076032.159989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076032.159993:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076032.160000:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3020 00000100:00080000:3.0:1769076032.160005:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076032.160010:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076032.160023:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076032.160044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612566784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076032.160062:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612566656:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076032.160065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076032.160077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076032.160080:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076032.160087:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3020 00000100:00080000:3.0:1769076032.160094:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076032.160098:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076032.160105:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076032.160124:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612566912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076032.160136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076032.160140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076032.160142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076032.160147:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3020 00000100:00080000:3.0:1769076032.160151:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076032.160154:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076032.160160:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076032.160173:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612567040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076032.160232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076032.160284:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076032.160311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076033.055777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076033.055898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076034.079795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076034.079919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076035.103787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076035.103962:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076035.232144:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076035.232212:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076035.232246:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076035.232333:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076035.232347:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076035.232361:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076035.232428:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076035] req@ffff8b10037dc700 x1855007612566784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076035.232479:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612566784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076035.232520:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076035.232529:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076035.232537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076035.232550:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076035] req@ffff8b10037df800 x1855007612566912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076035.232570:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612566912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076035.232605:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076035.232611:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076035.232616:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612566912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076035.232627:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076035] req@ffff8b10037df480 x1855007612567040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076035.232724:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612567040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076035.232738:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076035.232744:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076035.232749:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076035.232761:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076035] req@ffff8b10037dce00 x1855007612566656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076035.232807:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612566656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076035.232817:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076035.232822:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076035.232826:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612566656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076036.127746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076036.127843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076037.151719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076037.151816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076037.279856:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076037.279870:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076037.279876:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076037.279889:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3025 00000100:00080000:2.0:1769076037.279902:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076037.279911:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076037.279964:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612567168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076037.280017:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076037.280040:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076037.280044:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076037.280051:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3025 00000100:00080000:2.0:1769076037.280057:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076037.280063:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076037.280076:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076037.280122:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612567296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076037.280148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612567168:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076037.280159:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076037.280186:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076037.280190:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076037.280200:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3025 00000100:00080000:2.0:1769076037.280214:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076037.280222:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076037.280233:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076037.280259:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612567424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076037.280281:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076037.280290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076037.280293:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076037.280301:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076037.280312:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3025 00000100:00080000:2.0:1769076037.280323:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076037.280332:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076037.280344:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076037.280362:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff8b1005d59880 x1855007612567168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076037.280370:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612567552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076037.280445:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612567168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076037.280474:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076037.280484:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076037.280493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612567168:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076037.280590:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567296:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076037.280708:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076037.280742:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff8b1005d59500 x1855007612567296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076037.280770:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612567296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076037.280788:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076037.280797:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076037.280804:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567296:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076037.280816:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076037.280861:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076037.280886:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff8b1005d5aa00 x1855007612567424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076037.280913:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612567424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076037.280932:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076037.280950:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076037.280959:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076037.280970:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076037.281012:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076037.281035:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff8b1005d5b100 x1855007612567552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076037.281058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612567552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076037.281075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076037.281082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076037.281088:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076038.175837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076038.175894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076039.199774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076039.199864:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076040.223774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076040.223847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076041.247703:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076041.247770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076042.271756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076042.271870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076042.399796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076042.399814:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076042.399821:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076042.399851:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3030 00000100:00080000:1.0:1769076042.399867:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076042.399895:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076042.399948:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612567680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076042.400004:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076042.400011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076042.400015:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076042.400023:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3030 00000100:00080000:1.0:1769076042.400031:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076042.400039:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076042.400054:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076042.400073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612567680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076042.400077:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612567808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076042.400104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076042.400126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076042.400130:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076042.400136:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3030 00000100:00080000:1.0:1769076042.400142:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076042.400149:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076042.400157:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076042.400185:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612567936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076042.400201:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769076042.400203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076042.400208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076042.400211:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076042.400236:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3030 00000100:00080000:1.0:1769076042.400243:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076042.400250:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076042.400259:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076042.400268:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff8b1006135c00 x1855007612567680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076042.400279:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612568064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076042.400304:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612567680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076042.400324:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076042.400334:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076042.400342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612567680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076042.400395:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567808:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076042.400435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076042.400451:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff8b1006137b80 x1855007612567808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076042.400503:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612567808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076042.400518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076042.400522:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076042.400526:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076042.400533:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076042.400556:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076042.400569:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff8b1006135880 x1855007612567936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076042.400582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612567936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076042.400598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076042.400602:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076042.400606:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612567936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076042.400629:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076042.400650:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076042.400698:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff8b1006135500 x1855007612568064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076042.400712:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612568064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076042.400723:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076042.400738:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076042.400742:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076043.295787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076043.295824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076044.319759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076044.319906:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076045.343876:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076045.343922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076046.367753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076046.367920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076047.391725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076047.391896:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076047.519836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076047.519858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076047.519867:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076047.519887:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3035 00000100:00080000:3.0:1769076047.519906:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076047.519917:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076047.519981:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612568192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076047.520033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076047.520039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076047.520041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076047.520047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3035 00000100:00080000:3.0:1769076047.520051:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076047.520057:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076047.520070:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076047.520089:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612568320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076047.520103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076047.520108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076047.520111:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076047.520116:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3035 00000100:00080000:3.0:1769076047.520120:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076047.520127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076047.520134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612568192:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076047.520136:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076047.520157:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612568448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076047.520171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076047.520176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076047.520178:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076047.520184:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3035 00000100:00080000:3.0:1769076047.520189:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076047.520195:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076047.520202:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076047.520219:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612568576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076047.520260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076047.520330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076047] req@ffff8b10037ddc00 x1855007612568192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076047.520359:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612568192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076047.520401:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076047.520409:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076047.520415:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612568192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076047.520426:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568320:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076047.520461:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076047.520480:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076047] req@ffff8b10037df100 x1855007612568320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076047.520498:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612568320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076047.520510:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076047.520514:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076047.520518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076047.520553:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076047.520577:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076047.520590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076047] req@ffff8b10037dd880 x1855007612568448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076047.520606:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612568448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076047.520616:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076047.520619:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076047.520623:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076047.520630:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076047.520688:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076047.520703:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076047] req@ffff8b10037dea00 x1855007612568576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076047.520719:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612568576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076047.520738:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076047.520742:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076047.520747:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076048.415708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076048.415879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076049.439814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076049.439986:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076050.463789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076050.463873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076051.487743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076051.487810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076052.511878:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076052.511896:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076052.639748:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.639762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076052.639768:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.639779:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3040 00000100:00080000:3.0:1769076052.639796:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.639808:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076052.639866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612568704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.639921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.639929:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076052.639931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.639938:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3040 00000100:00080000:3.0:1769076052.639944:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.639950:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076052.639962:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076052.639981:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612568832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.639997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.640002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076052.640005:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.640010:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3040 00000100:00080000:3.0:1769076052.640016:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.640021:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076052.640026:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076052.640037:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612568960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.640051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.640060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769076052.640063:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612568704:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076052.640064:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.640074:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3040 00000100:00080000:3.0:1769076052.640084:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.640090:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076052.640096:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076052.640110:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612569088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076052.640174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076052.640221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff8b10037dc000 x1855007612568704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076052.640245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612568704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076052.640258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076052.640264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076052.640269:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612568704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076052.640279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568832:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076052.640339:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076052.640357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff8b10037dd500 x1855007612568832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076052.640373:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612568832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076052.640384:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076052.640389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076052.640393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568832:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076052.640424:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076052.640448:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076052.640460:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff8b10037ddf80 x1855007612568960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076052.640475:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612568960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076052.640486:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076052.640489:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076052.640493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612568960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076052.640500:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076052.640519:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076052.640531:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff8b10037de300 x1855007612569088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076052.640546:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612569088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076052.640555:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076052.640559:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076052.640562:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076053.535806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076053.535908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076054.559781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076054.559924:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076055.583808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076055.583911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076056.607780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076056.607904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076057.631794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076057.631975:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076057.759784:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076057.759803:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076057.759810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076057.759828:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3045 00000100:00080000:3.0:1769076057.759845:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076057.759858:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076057.759914:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612569216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076057.759979:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076057.759984:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076057.759987:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076057.759993:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3045 00000100:00080000:3.0:1769076057.759998:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076057.760004:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076057.760015:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076057.760033:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612569344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076057.760047:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076057.760051:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076057.760054:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076057.760058:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3045 00000100:00080000:3.0:1769076057.760062:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076057.760067:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076057.760073:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076057.760086:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612569216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076057.760088:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612569472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076057.760133:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076057.760137:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076057.760140:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076057.760145:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3045 00000100:00080000:3.0:1769076057.760151:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076057.760156:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076057.760162:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076057.760178:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612569600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076057.760248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076057.760313:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff8b10037dd180 x1855007612569216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076057.760340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612569216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076057.760354:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076057.760360:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076057.760366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612569216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076057.760376:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569344:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076057.760407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076057.760422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff8b10037ded80 x1855007612569344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076057.760438:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612569344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076057.760449:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076057.760453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076057.760458:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076057.760488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076057.760510:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076057.760523:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff8b10037de680 x1855007612569472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076057.760556:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612569472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076057.760573:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076057.760577:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076057.760581:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076057.760587:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076057.760606:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076057.760617:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff8b10037dc380 x1855007612569600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076057.760630:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612569600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076057.760657:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076057.760661:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076057.760664:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076058.655832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076058.655902:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076059.679857:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076059.679968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076060.703772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076060.703862:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076061.727854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076061.727948:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076062.751700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076062.751857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076062.879790:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076062.879801:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076062.879806:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076062.879818:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3050 00000100:00080000:2.0:1769076062.879830:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076062.879837:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076062.879876:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612569728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076062.879908:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076062.879912:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076062.879915:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076062.879919:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3050 00000100:00080000:2.0:1769076062.879923:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076062.879927:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076062.879946:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076062.879961:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612569856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076062.879967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612569728:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076062.879979:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076062.879984:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076062.879986:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076062.879992:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3050 00000100:00080000:2.0:1769076062.879996:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076062.880000:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076062.880006:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076062.880019:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612569984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076062.880032:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076062.880032:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076062.880039:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076062.880041:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076062.880047:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3050 00000100:00080000:2.0:1769076062.880052:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076062.880056:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076062.880061:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076062.880072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff8b1005d58380 x1855007612569728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076062.880084:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612570112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076062.880089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612569728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.880111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.880133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076062.880136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612569728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076062.880159:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569856:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076062.880178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076062.880188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff8b1005d5ad80 x1855007612569856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.880197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612569856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.880202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.880204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076062.880207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569856:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076062.880211:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076062.880223:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076062.880229:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff8b1005d5b100 x1855007612569984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.880237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612569984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.880242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.880244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076062.880246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612569984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076062.880249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076062.880260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076062.880266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff8b1005d5aa00 x1855007612570112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.880274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612570112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.880279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.880281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076062.880283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076063.775753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076063.775905:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076064.799781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076064.799860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076065.823793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076065.823884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076066.847858:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076066.847906:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076067.871752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076067.871867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076067.999842:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076067.999854:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076067.999860:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076067.999874:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3056 00000100:00080000:2.0:1769076067.999888:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076067.999898:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076067.999955:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612570240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076067.999993:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076068.000000:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076068.000003:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076068.000010:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3056 00000100:00080000:2.0:1769076068.000018:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076068.000025:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076068.000071:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076068.000089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612570240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076068.000100:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612570368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076068.000124:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076068.000129:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076068.000133:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076068.000139:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3056 00000100:00080000:2.0:1769076068.000146:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076068.000153:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076068.000162:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076068.000183:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612570496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076068.000194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076068.000212:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076068.000217:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076068.000220:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076068.000226:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3056 00000100:00080000:2.0:1769076068.000232:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076068.000238:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076068.000247:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076068.000259:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612570624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076068.000377:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076068] req@ffff8b1005d59500 x1855007612570240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076068.000404:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612570240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076068.000419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076068.000424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076068.000431:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612570240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076068.000461:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570368:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076068.000689:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076068.000719:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076068] req@ffff8b1005d59880 x1855007612570368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076068.000813:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612570368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076068.000828:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076068.000835:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076068.000841:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570368:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076068.000853:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076068.000885:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076068.000902:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076068/real 1769076068] req@ffff8b1005d5bb80 x1855007612570496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076068.000918:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612570496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076068.000929:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076068.000933:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076068.000937:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076068.000943:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076068.000963:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076068.000976:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076068/real 1769076068] req@ffff8b1005d58e00 x1855007612570624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076068.000991:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612570624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076068.001001:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076068.001005:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076068.001008:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076068.895767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076068.895945:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076069.919762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076069.919903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076070.943862:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076070.943971:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076071.967808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076071.967925:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076072.991774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076072.991851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076073.119858:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.119870:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076073.119876:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.119887:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3061 00000100:00080000:3.0:1769076073.119898:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.119904:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076073.119939:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612570752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.120006:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.120009:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.120011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.120015:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3061 00000100:00080000:3.0:1769076073.120018:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.120022:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076073.120031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.120046:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612570880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.120056:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.120060:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.120062:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.120065:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3061 00000100:00080000:3.0:1769076073.120068:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.120071:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076073.120076:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076073.120080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612570752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076073.120088:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612571008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.120098:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.120100:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076073.120102:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.120105:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3061 00000100:00080000:3.0:1769076073.120108:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.120112:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076073.120116:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.120125:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612571136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076073.120149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076073.120183:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff8b10037dca80 x1855007612570752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.120200:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612570752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.120210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.120214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076073.120218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612570752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076073.120223:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570880:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076073.120240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076073.120249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff8b10037dc380 x1855007612570880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.120259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612570880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.120271:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.120274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076073.120277:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612570880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076073.120298:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076073.120312:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076073.120319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff8b10037de680 x1855007612571008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.120329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612571008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.120336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.120338:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076073.120341:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076073.120345:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076073.120358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076073.120365:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff8b10037ded80 x1855007612571136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.120375:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612571136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.120381:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.120383:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076073.120386:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076074.015808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076074.015842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076075.039756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076075.039982:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076076.063802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076076.063923:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076077.087747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076077.087933:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076078.111740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076078.111795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076078.239841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076078.239875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076078.239881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076078.239894:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3066 00000100:00080000:1.0:1769076078.239908:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076078.239917:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076078.239966:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612571264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076078.240007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076078.240012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076078.240014:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076078.240019:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3066 00000100:00080000:1.0:1769076078.240023:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076078.240029:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076078.240040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076078.240057:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612571392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076078.240072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612571264:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076078.240077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076078.240090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076078.240093:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076078.240098:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3066 00000100:00080000:1.0:1769076078.240113:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076078.240120:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076078.240128:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076078.240144:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612571520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076078.240158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076078.240162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076078.240165:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076078.240174:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3066 00000800:00000100:3.0:1769076078.240176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769076078.240181:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076078.240188:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076078.240195:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076078.240212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612571648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076078.240241:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff8b1006137100 x1855007612571264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076078.240272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612571264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076078.240287:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076078.240293:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076078.240299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612571264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076078.240328:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571392:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076078.240355:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076078.240370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff8b1006134700 x1855007612571392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076078.240386:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612571392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076078.240397:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076078.240401:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076078.240405:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571392:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076078.240412:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076078.240433:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076078.240445:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff8b1006136d80 x1855007612571520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076078.240486:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612571520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076078.240497:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076078.240501:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076078.240505:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076078.240512:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076078.240535:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076078.240548:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff8b1006136680 x1855007612571648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076078.240562:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612571648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076078.240572:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076078.240576:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076078.240579:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076079.135789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076079.136012:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076080.159773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076080.159894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076081.183891:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076081.183898:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076082.207754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076082.207936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076083.231809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076083.231906:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076083.359866:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076083.359894:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076083.359899:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076083.359910:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3071 00000100:00080000:0.0:1769076083.359922:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076083.359930:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076083.359984:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612571776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076083.360010:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076083.360014:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076083.360016:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076083.360019:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3071 00000100:00080000:0.0:1769076083.360023:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076083.360026:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076083.360040:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076083.360051:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612571904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076083.360060:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076083.360064:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076083.360065:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076083.360068:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3071 00000100:00080000:0.0:1769076083.360072:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076083.360074:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076083.360079:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076083.360082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612571776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076083.360092:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612572032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076083.360101:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076083.360104:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076083.360106:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076083.360109:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3071 00000100:00080000:0.0:1769076083.360112:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076083.360115:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076083.360119:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076083.360129:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612572160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076083.360163:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076083.360209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff8b1006024a80 x1855007612571776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076083.360250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612571776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076083.360274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076083.360278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076083.360283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612571776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076083.360291:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571904:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076083.360317:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076083.360330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff8b1006025f80 x1855007612571904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076083.360342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612571904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076083.360350:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076083.360353:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076083.360356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612571904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076083.360379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076083.360396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076083.360410:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff8b1006027b80 x1855007612572032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076083.360427:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612572032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076083.360440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076083.360445:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076083.360450:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076083.360459:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076083.360502:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076083.360516:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff8b1006026d80 x1855007612572160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076083.360526:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612572160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076083.360533:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076083.360536:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076083.360539:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076084.255827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076084.255904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076085.279784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076085.279950:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076086.303752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076086.303930:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076087.327834:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076087.327923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076088.351896:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076088.351915:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076088.479842:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.479856:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076088.479862:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.479874:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3076 00000100:00080000:3.0:1769076088.479888:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.479898:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076088.479943:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612572288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.479986:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.479991:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076088.479994:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.479999:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3076 00000100:00080000:3.0:1769076088.480004:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.480010:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076088.480020:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076088.480036:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612572416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.480050:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.480056:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076088.480059:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.480063:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3076 00000100:00080000:3.0:1769076088.480067:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.480073:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076088.480079:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076088.480093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612572288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076088.480095:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612572544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.480122:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.480126:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076088.480128:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.480133:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3076 00000100:00080000:3.0:1769076088.480137:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.480142:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076088.480148:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076088.480164:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612572672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076088.480216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076088.480264:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff8b10037dd180 x1855007612572288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076088.480290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612572288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076088.480304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076088.480310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076088.480315:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612572288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076088.480329:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572416:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076088.480358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076088.480374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff8b10037de300 x1855007612572416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076088.480390:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612572416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076088.480400:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076088.480404:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076088.480408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572416:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076088.480437:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076088.480459:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076088.480472:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff8b10037ddf80 x1855007612572544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076088.480487:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612572544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076088.480497:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076088.480501:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076088.480504:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076088.480512:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076088.480531:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076088.480542:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff8b10037dd500 x1855007612572672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076088.480558:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612572672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076088.480568:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076088.480572:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076088.480575:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076089.375812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076089.375930:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076090.399794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076090.399882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076091.423792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076091.423894:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076092.447745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076092.447884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076093.471770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076093.471902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076093.599804:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076093.599818:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076093.599823:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076093.599835:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3081 00000100:00080000:3.0:1769076093.599850:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076093.599861:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076093.599912:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612572800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076093.599965:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076093.599973:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076093.599977:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076093.599986:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3081 00000100:00080000:3.0:1769076093.599991:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076093.599997:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076093.600008:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076093.600041:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612572928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076093.600053:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076093.600057:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076093.600059:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076093.600065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612572800:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076093.600066:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3081 00000100:00080000:3.0:1769076093.600074:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076093.600079:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076093.600084:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076093.600096:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612573056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076093.600126:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076093.600130:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076093.600133:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076093.600138:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3081 00000100:00080000:3.0:1769076093.600143:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076093.600147:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076093.600154:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076093.600167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076093.600173:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612573184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076093.600249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff8b10037dc000 x1855007612572800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076093.600269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612572800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076093.600279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076093.600283:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076093.600287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612572800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076093.600311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572928:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076093.600334:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076093.600345:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff8b10037dea00 x1855007612572928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076093.600357:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612572928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076093.600371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076093.600374:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076093.600377:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612572928:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076093.600382:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076093.600397:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076093.600405:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff8b10037dd880 x1855007612573056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076093.600416:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612573056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076093.600423:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076093.600425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076093.600428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076093.600433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076093.600447:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076093.600455:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff8b10037df100 x1855007612573184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076093.600466:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612573184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076093.600473:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076093.600475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076093.600479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076094.495744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076094.495855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076095.519825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076095.519843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076096.543712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076096.543866:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076097.567776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076097.567898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076098.591813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076098.591895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076098.719830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076098.719844:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076098.719851:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076098.719864:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3086 00000100:00080000:1.0:1769076098.719879:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076098.719890:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076098.719938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612573312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076098.719977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076098.719982:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076098.719985:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076098.719992:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3086 00000100:00080000:1.0:1769076098.719999:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076098.720009:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076098.720026:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076098.720033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612573312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076098.720058:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612573440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076098.720085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076098.720093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076098.720097:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076098.720106:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3086 00000100:00080000:1.0:1769076098.720116:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076098.720124:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076098.720133:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076098.720152:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612573568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076098.720167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076098.720172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076098.720175:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076098.720185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573440:192.168.204.102@tcp:38:kworker.0 00000100:00080000:1.0:1769076098.720186:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3086 00000100:00080000:1.0:1769076098.720198:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076098.720205:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076098.720212:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076098.720234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612573696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076098.720255:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076098.720296:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076099.615783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076099.615863:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076100.639782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076100.639938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076101.663790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076101.663936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076101.791941:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076101.791993:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076101.792015:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076101.792067:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076101.792083:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076101.792094:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769076101.792151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076101] req@ffff8b1006136680 x1855007612573696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076101.792188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612573696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076101.792205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076101.792211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076101.792217:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076101.792228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076101] req@ffff8b1006134a80 x1855007612573440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076101.792247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612573440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076101.792258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076101.792262:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076101.792267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573440:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076101.792278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076101] req@ffff8b1006137480 x1855007612573568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076101.792297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612573568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076101.792308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076101.792313:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076101.792317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076101.792325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076101] req@ffff8b1006134e00 x1855007612573312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076101.792342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612573312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076101.792352:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076101.792357:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076101.792362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612573312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076102.687862:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076102.687982:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076103.711733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076103.711848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076103.839843:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076103.839850:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076103.839854:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076103.839863:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3091 00000100:00080000:0.0:1769076103.839874:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076103.839880:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076103.839912:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612573824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076103.839935:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076103.839939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076103.839943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076103.839948:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3091 00000100:00080000:0.0:1769076103.839953:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076103.839959:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076103.839970:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076103.839974:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612573824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076103.839989:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612573952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076103.840004:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076103.840009:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076103.840011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076103.840016:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3091 00000100:00080000:0.0:1769076103.840029:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076103.840035:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076103.840041:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076103.840053:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076103.840058:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612574080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076103.840071:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076103.840076:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076103.840078:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076103.840083:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3091 00000100:00080000:0.0:1769076103.840087:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076103.840091:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076103.840098:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076103.840106:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff8b1006026300 x1855007612573824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076103.840112:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612574208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076103.840146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612573824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.840160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.840166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076103.840170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612573824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076103.840198:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573952:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076103.840225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076103.840240:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff8b1006024000 x1855007612573952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.840254:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612573952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.840264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.840267:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076103.840271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612573952:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076103.840278:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076103.840301:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076103.840314:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff8b1006025880 x1855007612574080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.840328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612574080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.840337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.840340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076103.840343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076103.840349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076103.840370:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076103.840382:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff8b1006026a00 x1855007612574208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.840395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612574208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.840412:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.840416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076103.840420:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076104.735719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076104.735884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076105.759737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076105.759805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076106.783752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076106.783910:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076107.807770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076107.807894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076108.831780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076108.831906:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076108.959816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.959833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076108.959841:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.959856:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3097 00000100:00080000:1.0:1769076108.959874:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.959886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076108.959937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612574336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.959983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.959992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076108.959998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.960008:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3097 00000100:00080000:1.0:1769076108.960015:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.960023:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076108.960040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076108.960054:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612574336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076108.960063:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612574464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.960082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.960088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076108.960091:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.960098:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3097 00000100:00080000:1.0:1769076108.960103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.960110:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076108.960118:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076108.960135:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612574592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076108.960143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769076108.960152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.960158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076108.960161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.960168:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3097 00000100:00080000:1.0:1769076108.960190:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.960197:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076108.960206:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076108.960211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff8b1006136d80 x1855007612574336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076108.960227:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612574720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076108.960244:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612574336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076108.960263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076108.960272:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076108.960280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612574336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076108.960318:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574464:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076108.960393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076108.960413:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff8b1006134700 x1855007612574464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076108.960430:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612574464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076108.960442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076108.960446:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076108.960450:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076108.960458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076108.960481:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076108.960493:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff8b1006137100 x1855007612574592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076108.960508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612574592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076108.960518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076108.960521:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076108.960525:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076108.960532:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076108.960552:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076108.960564:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff8b1006135500 x1855007612574720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076108.960578:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612574720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076108.960589:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076108.960592:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076108.960596:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076109.855843:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076109.855923:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076110.879740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076110.879879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076111.903745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076111.903938:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076112.927770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076112.927874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076113.951717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076113.951841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076114.079725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.079739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076114.079744:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.079757:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3102 00000100:00080000:1.0:1769076114.079771:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.079780:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076114.079840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612574848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.079869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.079873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076114.079875:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.079879:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3102 00000100:00080000:1.0:1769076114.079883:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.079886:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076114.079896:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076114.079908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612574976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.079917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.079921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076114.079923:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.079926:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3102 00000100:00080000:1.0:1769076114.079929:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.079935:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076114.079956:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612574848:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076114.079959:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076114.079975:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612575104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.079987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.079990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076114.079992:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.079995:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3102 00000100:00080000:1.0:1769076114.079999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.080003:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076114.080007:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076114.080017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612575232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076114.080045:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076114.080080:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076114] req@ffff8b1006135880 x1855007612574848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076114.080100:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612574848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076114.080110:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076114.080114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076114.080120:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612574848:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076114.080131:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574976:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076114.080161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076114.080195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076114] req@ffff8b1006137b80 x1855007612574976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076114.080214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612574976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076114.080225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076114.080228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076114.080232:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612574976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076114.080301:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076114.080332:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076114.080347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076114] req@ffff8b1006135c00 x1855007612575104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076114.080361:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612575104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076114.080371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076114.080374:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076114.080378:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076114.080384:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076114.080401:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076114.080411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076114] req@ffff8b1006136300 x1855007612575232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076114.080424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612575232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076114.080432:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076114.080435:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076114.080440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076114.975763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076114.975889:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076115.999721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076115.999897:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076117.023843:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076117.023903:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076118.047720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076118.047795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076119.071776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076119.071951:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076119.199802:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076119.199816:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076119.199821:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076119.199837:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3107 00000100:00080000:0.0:1769076119.199854:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076119.199867:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076119.199922:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612575360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076119.199981:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076119.199986:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076119.199989:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076119.199995:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3107 00000100:00080000:0.0:1769076119.200001:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076119.200008:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076119.200022:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076119.200053:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612575488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076119.200053:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612575360:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076119.200091:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076119.200103:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076119.200109:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076119.200118:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3107 00000100:00080000:0.0:1769076119.200127:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076119.200136:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076119.200148:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076119.200153:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076119.200207:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612575616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076119.200224:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076119.200229:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076119.200232:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076119.200242:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3107 00000100:00100000:3.0:1769076119.200245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff8b1006027800 x1855007612575360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076119.200249:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076119.200262:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076119.200278:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076119.200297:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612575744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076119.200323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612575360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.200351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076119.200358:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076119.200364:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612575360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076119.200398:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575488:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076119.200446:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076119.200466:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff8b1006027100 x1855007612575488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076119.200492:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612575488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.200509:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076119.200516:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076119.200524:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076119.200538:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076119.200582:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076119.200605:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff8b1006026680 x1855007612575616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076119.200621:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612575616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.200633:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076119.200637:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076119.200689:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076119.200699:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076119.200729:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076119.200745:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff8b1006025180 x1855007612575744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076119.200761:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612575744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.200772:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076119.200776:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076119.200781:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612575744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076120.095747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076120.095886:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076121.119719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076121.119851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076122.143797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076122.143879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076123.167798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076123.167926:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076124.191861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076124.191946:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076124.319849:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076124.319864:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076124.319871:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076124.319884:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3112 00000100:00080000:0.0:1769076124.319900:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076124.319911:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076124.319984:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612575872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076124.320017:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076124.320022:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076124.320025:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076124.320030:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3112 00000100:00080000:0.0:1769076124.320035:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076124.320040:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076124.320052:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076124.320075:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612576000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076124.320076:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612575872:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076124.320138:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076124.320144:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076124.320148:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076124.320154:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3112 00000100:00080000:0.0:1769076124.320159:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076124.320165:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076124.320173:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076124.320191:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612576128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076124.320212:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076124.320217:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076124.320223:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769076124.320226:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076124.320232:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3112 00000100:00080000:0.0:1769076124.320240:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076124.320246:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076124.320253:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076124.320269:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612576256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076124.320295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff8b1006025180 x1855007612575872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076124.320327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612575872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076124.320342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076124.320347:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076124.320353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612575872:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076124.320383:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576000:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076124.320436:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076124.320455:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff8b1006026680 x1855007612576000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076124.320473:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612576000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076124.320484:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076124.320489:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076124.320493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076124.320501:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076124.320524:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076124.320536:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff8b1006027100 x1855007612576128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076124.320552:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612576128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076124.320562:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076124.320566:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076124.320570:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076124.320576:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076124.320596:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076124.320608:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff8b1006027800 x1855007612576256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076124.320623:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612576256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076124.320678:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076124.320685:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076124.320693:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076125.215771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076125.215906:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076126.239720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076126.239849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076127.263755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076127.263853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076128.287788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076128.287937:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076129.311763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076129.311841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076129.439886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076129.439904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076129.439911:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076129.439928:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3117 00000100:00080000:3.0:1769076129.439945:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076129.439956:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076129.440040:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612576384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076129.440083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076129.440089:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076129.440092:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076129.440097:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3117 00000100:00080000:3.0:1769076129.440103:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076129.440108:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076129.440120:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076129.440137:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612576512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076129.440151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076129.440157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076129.440161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076129.440167:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3117 00000100:00080000:3.0:1769076129.440171:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076129.440177:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076129.440182:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076129.440195:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612576640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076129.440211:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612576384:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076129.440215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076129.440231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076129.440234:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076129.440239:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3117 00000100:00080000:3.0:1769076129.440256:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076129.440261:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076129.440267:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076129.440285:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612576768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076129.440334:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076129.440398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff8b10037ddc00 x1855007612576384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076129.440436:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612576384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076129.440456:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076129.440466:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076129.440474:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612576384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076129.440491:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576512:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076129.440548:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076129.440571:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff8b10037dce00 x1855007612576512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076129.440612:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612576512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076129.440630:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076129.440638:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076129.440688:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576512:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076129.440726:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076129.440761:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076129.440777:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff8b10037df480 x1855007612576640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076129.440794:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612576640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076129.440805:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076129.440809:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076129.440813:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076129.440820:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076129.440841:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076129.440853:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff8b10037df800 x1855007612576768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076129.440868:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612576768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076129.440879:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076129.440882:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076129.440886:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612576768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076130.335764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076130.335921:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076131.359864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076131.359952:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076132.383843:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076132.383982:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076133.407765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076133.407923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076134.431745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076134.431868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076134.559742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076134.559752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076134.559756:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076134.559764:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3122 00000100:00080000:3.0:1769076134.559775:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076134.559781:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076134.559816:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612576896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076134.559874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076134.559879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076134.559882:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076134.559887:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3122 00000100:00080000:3.0:1769076134.559892:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076134.559897:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076134.559908:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076134.559952:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612577024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076134.559967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076134.559972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076134.559975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076134.559981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3122 00000100:00080000:3.0:1769076134.559986:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076134.559995:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076134.559997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612576896:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076134.560003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076134.560020:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612577152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076134.560035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076134.560040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076134.560042:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076134.560047:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3122 00000100:00080000:3.0:1769076134.560053:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076134.560058:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076134.560064:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076134.560110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076134.560111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612577280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076134.560156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff8b10037dc700 x1855007612576896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076134.560174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612576896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076134.560183:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076134.560187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076134.560190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612576896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076134.560196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577024:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076134.560214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076134.560224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff8b10037df800 x1855007612577024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076134.560234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612577024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076134.560241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076134.560243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076134.560246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577024:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076134.560266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076134.560280:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076134.560287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff8b10037df480 x1855007612577152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076134.560297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612577152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076134.560303:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076134.560305:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076134.560307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076134.560311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076134.560323:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076134.560330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff8b10037dce00 x1855007612577280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076134.560339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612577280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076134.560350:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076134.560352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076134.560355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076135.455816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076135.456016:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076136.479776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076136.479957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076137.503797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076137.503931:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076138.527867:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076138.527899:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076139.551757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076139.551979:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076139.679878:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076139.679896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076139.679905:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076139.679921:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3127 00000100:00080000:0.0:1769076139.679939:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076139.679951:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076139.680013:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612577408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076139.680057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076139.680061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076139.680064:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076139.680070:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3127 00000100:00080000:0.0:1769076139.680077:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076139.680086:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076139.680104:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076139.680129:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612577536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076139.680144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612577408:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076139.680152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076139.680164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076139.680169:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076139.680180:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3127 00000100:00080000:0.0:1769076139.680195:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076139.680206:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076139.680220:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076139.680256:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612577664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076139.680278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076139.680281:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076139.680293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076139.680299:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076139.680312:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3127 00000100:00080000:0.0:1769076139.680322:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076139.680331:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076139.680343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076139.680357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff8b1006026a00 x1855007612577408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076139.680367:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612577792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076139.680547:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612577408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.680566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.680573:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076139.680579:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612577408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076139.680615:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577536:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076139.680698:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076139.680723:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff8b1006025880 x1855007612577536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.680953:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612577536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.680967:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.680973:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076139.680980:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076139.681057:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076139.681098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076139.681124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff8b1006024000 x1855007612577664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.681141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612577664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.681152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.681157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076139.681161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076139.681168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076139.681190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076139.681202:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff8b1006026300 x1855007612577792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.681217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612577792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.681228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.681231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076139.681235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612577792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076140.575735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076140.576033:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076141.599769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076141.599910:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076142.623747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076142.623943:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076143.647789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076143.647937:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076144.671828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076144.671960:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076144.799833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076144.799846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076144.799852:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076144.799864:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3132 00000100:00080000:0.0:1769076144.799878:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076144.799887:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076144.799940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612577920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076144.799989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076144.799997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076144.800001:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076144.800010:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3132 00000100:00080000:0.0:1769076144.800019:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076144.800029:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076144.800051:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612577920:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076144.800058:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076144.800087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612578048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076144.800111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076144.800116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076144.800119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076144.800125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3132 00000100:00080000:0.0:1769076144.800132:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076144.800137:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076144.800144:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076144.800155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076144.800164:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612578176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076144.800180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076144.800184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076144.800186:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076144.800193:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3132 00000100:00080000:0.0:1769076144.800198:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076144.800203:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076144.800210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff8b1006026d80 x1855007612577920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076144.800214:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076144.800238:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612578304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076144.800251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612577920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076144.800272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076144.800279:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076144.800285:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612577920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076144.800321:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578048:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076144.800353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076144.800376:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff8b1006027b80 x1855007612578048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076144.800393:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612578048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076144.800405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076144.800409:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076144.800413:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578048:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076144.800421:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076144.800448:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076144.800471:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff8b1006025f80 x1855007612578176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076144.800506:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612578176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076144.800537:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076144.800545:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076144.800571:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076144.800586:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076144.800675:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076144.800700:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff8b1006024a80 x1855007612578304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076144.800717:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612578304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076144.800728:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076144.800732:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076144.800737:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076145.695889:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076145.695933:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076146.719816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076146.719903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076147.743832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076147.743934:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076148.767780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076148.767895:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076149.791765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076149.791918:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076149.919829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076149.919842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076149.919848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076149.919860:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3138 00000100:00080000:3.0:1769076149.919874:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076149.919883:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076149.919931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612578432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076149.919974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076149.919981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076149.919983:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076149.919989:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3138 00000100:00080000:3.0:1769076149.919994:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076149.920000:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076149.920013:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076149.920031:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612578560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076149.920040:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612578432:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076149.920057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076149.920064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076149.920067:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076149.920074:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3138 00000100:00080000:3.0:1769076149.920079:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076149.920083:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076149.920091:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076149.920108:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612578688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076149.920133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076149.920140:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076149.920143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076149.920149:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076149.920157:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3138 00000100:00080000:3.0:1769076149.920165:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076149.920171:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076149.920179:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076149.920204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff8b10037ddc00 x1855007612578432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076149.920205:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612578816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076149.920242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612578432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076149.920260:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076149.920268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076149.920293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612578432:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076149.920324:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578560:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076149.920353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076149.920370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff8b10037df100 x1855007612578560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076149.920387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612578560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076149.920397:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076149.920402:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076149.920406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076149.920413:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076149.920435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076149.920447:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff8b10037dd880 x1855007612578688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076149.920461:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612578688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076149.920471:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076149.920475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076149.920478:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076149.920485:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076149.920504:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076149.920517:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff8b10037dea00 x1855007612578816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076149.920531:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612578816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076149.920541:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076149.920545:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076149.920548:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612578816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076150.815812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076150.815867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076151.839804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076151.839901:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076152.863890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076152.863927:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076153.887773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076153.887854:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076154.911789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076154.911962:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076155.039828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076155.039841:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076155.039847:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076155.039857:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3143 00000100:00080000:0.0:1769076155.039870:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076155.039878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076155.039921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612578944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076155.039956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076155.039961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076155.039964:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076155.039970:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3143 00000100:00080000:0.0:1769076155.039976:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076155.039982:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076155.039993:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612578944:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076155.039994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076155.040017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612579072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076155.040043:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076155.040048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076155.040051:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076155.040056:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3143 00000100:00080000:0.0:1769076155.040061:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076155.040066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076155.040073:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076155.040090:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612579200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076155.040099:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076155.040116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076155.040120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076155.040123:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076155.040129:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3143 00000100:00080000:0.0:1769076155.040135:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076155.040140:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076155.040147:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff8b1006024e00 x1855007612578944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076155.040149:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076155.040171:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612579328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076155.040176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612578944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076155.040197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076155.040201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076155.040204:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612578944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076155.040232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579072:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076155.040264:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076155.040275:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff8b1006024380 x1855007612579072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076155.040285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612579072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076155.040292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076155.040294:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076155.040297:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579072:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076155.040301:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076155.040316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076155.040323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff8b1006027480 x1855007612579200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076155.040332:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612579200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076155.040348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076155.040350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076155.040352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076155.040356:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076155.040368:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076155.040375:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076155/real 1769076155] req@ffff8b1006024700 x1855007612579328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076155.040385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612579328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076155.040391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076155.040393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076155.040395:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076155.935712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076155.935886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076156.959742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076156.959915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076157.983827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076157.983880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076159.007774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076159.007857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076160.031915:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076160.031925:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076160.159758:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076160.159771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076160.159777:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076160.159788:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3148 00000100:00080000:0.0:1769076160.159801:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076160.159808:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076160.159862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612579456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076160.159898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076160.159902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076160.159904:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076160.159907:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3148 00000100:00080000:0.0:1769076160.159911:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076160.159914:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076160.159924:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076160.159936:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612579584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076160.159944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076160.159947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076160.159966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076160.159969:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3148 00000100:00080000:0.0:1769076160.159974:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076160.159978:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076160.159983:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076160.159992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612579712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076160.160002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076160.160007:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076160.160009:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076160.160010:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612579456:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076160.160024:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3148 00000100:00080000:0.0:1769076160.160029:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076160.160044:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076160.160049:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076160.160056:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612579840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076160.160159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076160.160199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff8b1006024700 x1855007612579456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076160.160244:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612579456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076160.160256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076160.160261:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076160.160265:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612579456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076160.160273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579584:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076160.160296:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076160.160308:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff8b1006027480 x1855007612579584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076160.160320:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612579584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076160.160346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076160.160349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076160.160353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579584:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076160.160380:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076160.160422:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076160.160428:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff8b1006024380 x1855007612579712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076160.160436:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612579712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076160.160445:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076160.160447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076160.160449:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076160.160453:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076160.160479:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076160.160514:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff8b1006024e00 x1855007612579840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076160.160524:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612579840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076160.160532:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076160.160534:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076160.160537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612579840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076161.055822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076161.055896:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076162.079813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076162.079936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076163.103804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076163.103976:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076164.127762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076164.127869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076165.151760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076165.151981:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076165.279808:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076165.279823:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076165.279829:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076165.279842:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3153 00000100:00080000:2.0:1769076165.279857:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076165.279867:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076165.279932:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612579968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076165.279980:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076165.279987:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076165.279991:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076165.279999:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3153 00000100:00080000:2.0:1769076165.280021:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076165.280033:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076165.280051:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076165.280053:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612579968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076165.280103:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612580096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076165.280127:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076165.280131:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076165.280134:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076165.280138:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3153 00000100:00080000:2.0:1769076165.280163:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076165.280169:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076165.280177:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076165.280197:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612580224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076165.280210:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076165.280216:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076165.280219:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076165.280225:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3153 00000100:00100000:3.0:1769076165.280251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580096:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769076165.280282:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076165.280291:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076165.280301:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076165.280325:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076165.280328:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612580352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076165.280454:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076166.175795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076166.175867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076167.199851:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076167.199923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076168.223778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076168.223898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076168.351901:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076168.351950:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076168.351980:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076168.352026:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076168.352039:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076168.352053:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076168.352109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076168] req@ffff8b1005d5a300 x1855007612580352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076168.352149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612580352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076168.352172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076168.352179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076168.352184:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076168.352193:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076168] req@ffff8b1005d5b480 x1855007612580096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076168.352210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612580096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076168.352231:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076168.352236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076168.352241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580096:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076168.352250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076168] req@ffff8b1005d59180 x1855007612580224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076168.352267:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612580224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076168.352278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076168.352283:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076168.352287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076168.352295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076168] req@ffff8b1005d5b800 x1855007612579968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076168.352317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612579968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076168.352327:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076168.352332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076168.352336:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612579968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076169.247807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076169.247905:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076170.271725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076170.271824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076170.399752:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076170.399764:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076170.399770:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076170.399781:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3158 00000100:00080000:2.0:1769076170.399794:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076170.399804:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076170.399854:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612580480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076170.399908:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076170.399915:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076170.399918:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076170.399926:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3158 00000100:00080000:2.0:1769076170.399933:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076170.399939:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076170.399954:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076170.399976:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612580608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076170.399996:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076170.400021:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076170.400025:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076170.400032:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3158 00000100:00080000:2.0:1769076170.400039:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076170.400046:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076170.400057:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076170.400079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612580480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076170.400105:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612580736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076170.400132:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076170.400154:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076170.400157:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076170.400163:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3158 00000100:00080000:2.0:1769076170.400172:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076170.400178:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076170.400186:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076170.400205:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612580864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076170.400222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076170.400284:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff8b1005d58a80 x1855007612580480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.400342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612580480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.400359:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.400366:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076170.400373:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612580480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076170.400387:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580608:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076170.400451:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076170.400492:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff8b1005d58000 x1855007612580608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.400517:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612580608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.400531:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.400537:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076170.400544:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076170.400582:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076170.400620:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076170.400643:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff8b1005d5b800 x1855007612580736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.400667:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612580736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.400733:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.400740:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076170.400746:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076170.400758:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076170.400796:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076170.400815:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff8b1005d59180 x1855007612580864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.400837:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612580864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.400853:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.400857:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076170.400862:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612580864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076171.295836:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076171.295890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076172.319703:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076172.319803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076173.343724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076173.343885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076174.367843:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076174.367851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076175.391709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076175.391825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076175.519741:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076175.519748:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076175.519752:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076175.519761:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3163 00000100:00080000:2.0:1769076175.519772:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076175.519780:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076175.519822:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612580992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076175.519854:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076175.519858:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076175.519860:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076175.519864:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3163 00000100:00080000:2.0:1769076175.519871:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076175.519877:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076175.519887:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076175.519901:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612580992:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076175.519903:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612581120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076175.519917:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076175.519920:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076175.519922:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076175.519925:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3163 00000100:00080000:2.0:1769076175.519928:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076175.519931:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076175.519935:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076175.519944:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612581248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076175.519951:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076175.519954:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076175.519956:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076175.519958:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3163 00000100:00080000:2.0:1769076175.519961:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076175.519963:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769076175.519968:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769076175.519972:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076175.519982:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612581376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076175.520004:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff8b1005d5b480 x1855007612580992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076175.520024:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612580992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076175.520039:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076175.520043:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076175.520049:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612580992:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076175.520072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581120:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076175.520095:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076175.520110:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff8b1005d5a300 x1855007612581120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076175.520124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612581120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076175.520134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076175.520137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076175.520141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076175.520148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076175.520268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076175.520278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff8b1005d58e00 x1855007612581248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076175.520286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612581248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076175.520296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076175.520299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076175.520301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076175.520305:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076175.520316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076175.520323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff8b1005d5bb80 x1855007612581376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076175.520330:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612581376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076175.520337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076175.520340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076175.520343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076176.415819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076176.415849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076177.439833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076177.439916:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076178.463781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076178.463896:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076179.487773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076179.487968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076180.511797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076180.511922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076180.639797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.639811:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076180.639817:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.639829:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3168 00000100:00080000:3.0:1769076180.639844:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.639853:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076180.639900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612581504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.639943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.639952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076180.639956:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.639963:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3168 00000100:00080000:3.0:1769076180.639970:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.639977:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076180.639992:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076180.640017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612581632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076180.640022:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612581504:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076180.640038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.640044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076180.640047:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.640054:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3168 00000100:00080000:3.0:1769076180.640060:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.640066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076180.640075:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076180.640091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612581760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.640107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.640113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769076180.640116:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076180.640117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.640123:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3168 00000100:00080000:3.0:1769076180.640129:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.640135:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076180.640173:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076180.640193:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612581888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076180.640195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076180] req@ffff8b10037dc000 x1855007612581504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076180.640230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612581504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076180.640264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076180.640274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076180.640282:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612581504:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076180.640315:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581632:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076180.640351:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076180.640369:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076180] req@ffff8b10037dd500 x1855007612581632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076180.640400:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612581632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076180.640411:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076180.640415:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076180.640420:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581632:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076180.640427:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076180.640453:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076180.640466:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076180] req@ffff8b10037ddf80 x1855007612581760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076180.640481:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612581760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076180.640504:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076180.640511:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076180.640517:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076180.640529:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076180.640570:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076180.640594:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076180] req@ffff8b10037de300 x1855007612581888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076180.640620:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612581888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076180.640637:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076180.640679:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076180.640683:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612581888:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769076181.535856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076181.535862:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076182.559819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076182.559871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076183.583761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076183.583853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076184.607784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076184.607963:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076185.631729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076185.631788:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076185.759786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076185.759795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076185.759798:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076185.759806:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3173 00000100:00080000:0.0:1769076185.759815:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076185.759821:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076185.759853:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612582016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076185.759877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076185.759880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076185.759881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076185.759884:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3173 00000100:00080000:0.0:1769076185.759886:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076185.759889:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076185.759895:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076185.759904:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612582144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076185.759911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076185.759913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076185.759914:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076185.759916:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3173 00000100:00080000:0.0:1769076185.759918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076185.759920:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076185.759923:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076185.759929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612582272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076185.759936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076185.759938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076185.759940:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076185.759943:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3173 00000100:00080000:0.0:1769076185.759944:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076185.759947:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076185.759950:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076185.759969:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612582016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076185.759970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612582400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076185.760032:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076185.760081:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff8b1006024a80 x1855007612582016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.760096:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612582016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.760104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.760108:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076185.760111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612582016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076185.760116:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582144:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076185.760134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076185.760143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff8b1006025f80 x1855007612582144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.760151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612582144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.760156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.760159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076185.760161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582144:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076185.760165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076185.760176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076185.760182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff8b1006027b80 x1855007612582272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.760189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612582272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.760195:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.760196:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076185.760198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076185.760218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076185.760230:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076185.760237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff8b1006026d80 x1855007612582400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.760245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612582400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.760250:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.760252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076185.760254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076186.655768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076186.655951:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076187.679796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076187.679927:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076188.703765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076188.703795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076189.727804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076189.727904:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076190.751770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076190.752010:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076190.879810:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.879825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076190.879832:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.879859:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3178 00000100:00080000:3.0:1769076190.879876:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.879888:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076190.879957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612582528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.880053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.880059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076190.880062:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.880068:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3178 00000100:00080000:3.0:1769076190.880074:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.880089:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076190.880101:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076190.880125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612582656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.880141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.880146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076190.880148:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.880153:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3178 00000100:00080000:3.0:1769076190.880159:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.880175:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076190.880176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612582528:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076190.880191:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076190.880210:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612582784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.880228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.880233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076190.880237:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.880246:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3178 00000100:00080000:3.0:1769076190.880265:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.880272:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076190.880284:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076190.880307:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612582912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076190.880327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076190.880390:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff8b10037dd180 x1855007612582528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.880410:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612582528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.880420:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.880425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076190.880429:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612582528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076190.880436:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582656:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076190.880455:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076190.880465:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff8b10037ded80 x1855007612582656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.880474:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612582656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.880486:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.880489:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076190.880492:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582656:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076190.880519:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076190.880534:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076190.880540:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff8b10037de680 x1855007612582784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.880549:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612582784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.880556:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.880558:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076190.880560:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076190.880565:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076190.880578:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076190.880585:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff8b10037dc380 x1855007612582912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.880594:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612582912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.880600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.880602:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076190.880605:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612582912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076191.775817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076191.775941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076192.799780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076192.799850:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076193.823803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076193.823992:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076194.847769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076194.847941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076195.871926:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076195.871957:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076195.999757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076195.999771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076195.999776:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076195.999789:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3184 00000100:00080000:3.0:1769076195.999803:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076195.999811:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076195.999857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612583040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076195.999945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076195.999959:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076195.999963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076195.999972:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3184 00000100:00080000:3.0:1769076195.999980:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076195.999988:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076196.000004:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076196.000029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612583040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076196.000031:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612583168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076196.000052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.000058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076196.000061:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.000069:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3184 00000100:00080000:3.0:1769076196.000075:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.000082:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076196.000091:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076196.000108:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612583296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076196.000122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076196.000125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076196.000131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076196.000134:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076196.000140:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3184 00000100:00080000:3.0:1769076196.000146:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076196.000153:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076196.000161:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076196.000179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076196] req@ffff8b10037dca80 x1855007612583040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076196.000179:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612583424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076196.000213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612583040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076196.000233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076196.000242:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076196.000251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612583040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076196.000288:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583168:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076196.000325:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076196.000343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076196] req@ffff8b10037dc380 x1855007612583168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076196.000359:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612583168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076196.000371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076196.000375:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076196.000379:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076196.000387:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076196.000409:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076196.000422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076196] req@ffff8b10037de680 x1855007612583296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076196.000437:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612583296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076196.000447:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076196.000450:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076196.000454:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076196.000461:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076196.000501:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076196.000515:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076196] req@ffff8b10037ded80 x1855007612583424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076196.000530:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612583424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076196.000540:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076196.000544:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076196.000547:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076196.895781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076196.895932:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076197.919760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076197.919943:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076198.943761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076198.943940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076199.967758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076199.967859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076200.991815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076200.991890:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076201.119732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.119745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076201.119751:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.119763:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3189 00000100:00080000:3.0:1769076201.119777:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.119786:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076201.119882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612583552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.119928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.119935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.119939:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.119948:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3189 00000100:00080000:3.0:1769076201.119956:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.119981:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076201.119997:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.120021:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612583680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.120038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076201.120064:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612583552:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076201.120064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.120068:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.120082:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3189 00000100:00080000:3.0:1769076201.120090:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.120095:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076201.120103:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.120121:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612583808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.120134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.120138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076201.120141:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.120145:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3189 00000100:00080000:3.0:1769076201.120149:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.120153:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076201.120159:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.120172:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612583936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076201.120184:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076201.120258:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff8b10037dd180 x1855007612583552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.120284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612583552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.120299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.120304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076201.120309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612583552:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076201.120338:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583680:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076201.120381:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076201.120397:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff8b10037de300 x1855007612583680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.120412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612583680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.120422:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.120426:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076201.120430:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076201.120437:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076201.120457:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076201.120468:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff8b10037ddf80 x1855007612583808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.120482:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612583808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.120491:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.120494:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076201.120497:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076201.120505:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076201.120523:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076201.120534:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff8b10037dd500 x1855007612583936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.120547:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612583936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.120556:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.120560:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076201.120563:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612583936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076202.015706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076202.015831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076203.039886:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076203.039935:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076204.063777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076204.063951:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076205.087759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076205.087825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076206.111873:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076206.111907:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076206.239855:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076206.239869:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076206.239875:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076206.239886:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3194 00000100:00080000:2.0:1769076206.239901:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076206.239910:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076206.239971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612584064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076206.240007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076206.240012:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076206.240015:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076206.240020:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3194 00000100:00080000:2.0:1769076206.240025:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076206.240030:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076206.240042:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076206.240060:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612584192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076206.240074:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076206.240081:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076206.240083:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076206.240087:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3194 00000100:00080000:2.0:1769076206.240091:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076206.240095:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076206.240103:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076206.240115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612584064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076206.240122:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612584320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076206.240148:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076206.240152:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076206.240155:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076206.240163:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3194 00000100:00080000:2.0:1769076206.240169:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076206.240173:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076206.240180:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076206.240193:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612584448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076206.240256:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076206.240324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff8b1005d59880 x1855007612584064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076206.240366:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612584064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076206.240391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076206.240401:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076206.240409:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612584064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076206.240426:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584192:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076206.240466:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076206.240484:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff8b1005d59500 x1855007612584192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076206.240504:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612584192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076206.240515:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076206.240520:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076206.240524:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584192:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076206.240563:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076206.240587:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076206.240600:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff8b1005d5aa00 x1855007612584320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076206.240616:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612584320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076206.240627:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076206.240631:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076206.240634:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076206.240738:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076206.240774:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076206.240791:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff8b1005d5b100 x1855007612584448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076206.240807:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612584448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076206.240827:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076206.240832:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076206.240836:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076207.135789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076207.135927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076208.159803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076208.159930:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076209.183805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076209.183920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076210.207875:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076210.207911:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076211.231799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076211.231900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076211.359829:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076211.359843:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076211.359849:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076211.359862:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3199 00000100:00080000:2.0:1769076211.359877:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076211.359888:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076211.359950:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612584576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076211.360001:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076211.360006:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076211.360009:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076211.360014:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3199 00000100:00080000:2.0:1769076211.360021:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076211.360029:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076211.360043:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076211.360068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612584576:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076211.360069:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612584704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076211.360098:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076211.360104:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076211.360106:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076211.360112:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3199 00000100:00080000:2.0:1769076211.360117:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076211.360122:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076211.360130:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076211.360148:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612584832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076211.360161:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076211.360165:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076211.360172:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076211.360173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076211.360185:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3199 00000100:00080000:2.0:1769076211.360194:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076211.360199:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076211.360207:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076211.360225:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612584960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076211.360232:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff8b1005d5ad80 x1855007612584576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076211.360274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612584576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076211.360290:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076211.360297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076211.360304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612584576:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076211.360337:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584704:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076211.360366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076211.360382:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff8b1005d58380 x1855007612584704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076211.360416:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612584704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076211.360428:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076211.360435:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076211.360440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584704:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076211.360448:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076211.360478:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076211.360496:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff8b1005d5b100 x1855007612584832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076211.360512:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612584832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076211.360524:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076211.360530:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076211.360536:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076211.360544:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076211.360574:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076211.360591:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff8b1005d5aa00 x1855007612584960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076211.360610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612584960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076211.360715:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076211.360723:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076211.360729:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612584960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076212.255824:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076212.255903:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076213.279756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076213.279885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076214.303704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076214.303791:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076215.327767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076215.327899:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076216.351757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076216.351854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076216.479857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076216.479874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076216.479892:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076216.479907:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3204 00000100:00080000:3.0:1769076216.479924:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076216.479936:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076216.479992:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612585088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076216.480054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076216.480063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076216.480066:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076216.480072:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3204 00000100:00080000:3.0:1769076216.480078:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076216.480085:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076216.480098:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076216.480119:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612585216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076216.480139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612585088:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076216.480139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076216.480158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076216.480161:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076216.480167:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3204 00000100:00080000:3.0:1769076216.480171:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076216.480175:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076216.480181:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076216.480207:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612585344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076216.480219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076216.480224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076216.480227:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076216.480232:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3204 00000100:00080000:3.0:1769076216.480237:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076216.480241:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076216.480248:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076216.480265:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612585472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076216.480271:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076216.480337:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff8b10037dc000 x1855007612585088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076216.480363:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612585088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076216.480378:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076216.480383:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076216.480389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612585088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076216.480418:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585216:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076216.480446:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076216.480462:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff8b10037dea00 x1855007612585216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076216.480478:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612585216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076216.480489:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076216.480493:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076216.480497:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076216.480505:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076216.480526:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076216.480570:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff8b10037dd880 x1855007612585344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076216.480599:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612585344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076216.480610:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076216.480614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076216.480618:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076216.480626:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076216.480714:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076216.480730:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff8b10037df100 x1855007612585472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076216.480752:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612585472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076216.480763:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076216.480767:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076216.480771:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585472:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769076217.375855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076217.375859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076218.399769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076218.399868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076219.423744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076219.423868:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076220.447753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076220.447851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076221.471757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076221.471936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076221.599909:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076221.599924:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076221.599931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076221.599943:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3209 00000100:00080000:2.0:1769076221.599958:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076221.599967:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076221.600017:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612585600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076221.600059:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076221.600065:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076221.600067:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076221.600075:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3209 00000100:00080000:2.0:1769076221.600079:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076221.600084:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076221.600096:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076221.600112:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612585728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076221.600128:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076221.600138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612585600:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076221.600139:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076221.600153:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076221.600168:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3209 00000100:00080000:2.0:1769076221.600173:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076221.600177:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076221.600184:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076221.600201:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612585856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076221.600214:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076221.600219:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076221.600221:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076221.600236:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3209 00000100:00080000:2.0:1769076221.600240:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076221.600245:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076221.600250:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076221.600260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076221.600269:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612585984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076221.600318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff8b1005d59500 x1855007612585600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076221.600353:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612585600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076221.600366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076221.600373:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076221.600378:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612585600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076221.600408:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585728:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076221.600437:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076221.600525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff8b1005d59880 x1855007612585728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076221.600554:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612585728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076221.600572:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076221.600579:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076221.600586:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585728:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076221.600597:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076221.600630:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076221.600699:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff8b1005d5bb80 x1855007612585856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076221.600716:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612585856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076221.600727:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076221.600732:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076221.600736:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076221.600743:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076221.600771:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076221.600785:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff8b1005d58e00 x1855007612585984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076221.600799:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612585984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076221.600809:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076221.600812:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076221.600817:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612585984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076222.495813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076222.495900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076223.519747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076223.519824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076224.543901:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076224.543930:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076225.567820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076225.567959:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076226.591786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076226.591918:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076226.719783:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076226.719799:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076226.719805:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076226.719821:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3214 00000100:00080000:2.0:1769076226.719837:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076226.719848:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076226.719921:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612586112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076226.719969:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076226.719976:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076226.719979:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076226.719985:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3214 00000100:00080000:2.0:1769076226.719991:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076226.720000:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076226.720016:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076226.720043:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612586240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076226.720083:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076226.720089:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076226.720095:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076226.720104:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3214 00000100:00080000:2.0:1769076226.720111:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076226.720123:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612586112:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076226.720127:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076226.720154:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076226.720205:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612586368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076226.720237:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076226.720243:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076226.720247:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076226.720257:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3214 00000100:00080000:2.0:1769076226.720265:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076226.720272:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769076226.720290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769076226.720298:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076226.720329:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612586496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076226.720402:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff8b1005d5a300 x1855007612586112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076226.720434:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612586112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076226.720451:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076226.720458:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076226.720464:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612586112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076226.720475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586240:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076226.720508:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076226.720525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff8b1005d5b480 x1855007612586240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076226.720541:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612586240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076226.720552:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076226.720556:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076226.720560:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586240:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076226.720603:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076226.720628:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076226.720687:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff8b1005d59180 x1855007612586368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076226.720705:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612586368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076226.720717:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076226.720721:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076226.720726:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076226.720734:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076226.720760:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076226.720786:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff8b1005d5b800 x1855007612586496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076226.720801:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612586496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076226.720811:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076226.720815:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076226.720818:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076227.615796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076227.615893:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076228.639752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076228.639909:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076229.663714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076229.663820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076230.687802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076230.687945:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076231.711899:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076231.711902:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076231.839769:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076231.839783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076231.839788:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076231.839801:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3219 00000100:00080000:2.0:1769076231.839818:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076231.839826:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076231.839873:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612586624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076231.839912:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076231.839917:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076231.839919:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076231.839924:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3219 00000100:00080000:2.0:1769076231.839928:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076231.839933:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076231.839945:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076231.839961:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612586752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076231.839977:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076231.839983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076231.839985:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076231.839996:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612586624:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076231.839997:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3219 00000100:00080000:2.0:1769076231.840004:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076231.840014:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076231.840024:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076231.840041:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612586880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076231.840053:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076231.840059:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076231.840062:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076231.840067:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3219 00000100:00080000:2.0:1769076231.840072:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076231.840076:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076231.840081:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076231.840098:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612587008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076231.840166:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076231.840204:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076231.840232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076232.735739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076232.735863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076233.759751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076233.759979:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076234.783729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076234.783819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076234.911990:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076234.912047:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076234.912069:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076234.912118:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076234.912129:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076234.912138:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769076234.912282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076234] req@ffff8b1005d58a80 x1855007612586752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076234.912329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612586752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076234.912350:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076234.912360:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076234.912369:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076234.912383:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076234] req@ffff8b1005d5b800 x1855007612586880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076234.912408:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612586880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076234.912425:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076234.912433:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076234.912440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612586880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076234.912452:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076234] req@ffff8b1005d59180 x1855007612587008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076234.912491:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612587008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076234.912506:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076234.912513:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076234.912521:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076234.912533:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076234] req@ffff8b1005d58000 x1855007612586624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076234.912559:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612586624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076234.912576:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076234.912584:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076234.912591:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612586624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076235.807776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076235.807864:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076236.831747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076236.831929:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076236.959872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.959885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076236.959890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.959903:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3225 00000100:00080000:1.0:1769076236.959926:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.959935:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076236.959978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612587136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076236.960025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.960034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076236.960038:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.960047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3225 00000100:00080000:1.0:1769076236.960054:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.960061:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076236.960078:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076236.960106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612587264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076236.960116:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612587136:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076236.960149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.960157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076236.960162:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.960172:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3225 00000100:00080000:1.0:1769076236.960182:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.960217:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076236.960232:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076236.960247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076236.960273:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612587392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076236.960305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076236.960312:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff8b1006137800 x1855007612587136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076236.960316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076236.960326:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.960348:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3225 00000100:00080000:1.0:1769076236.960358:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076236.960365:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612587136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076236.960374:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076236.960394:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076236.960401:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076236.960416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076236.960428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612587520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076236.960436:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612587136:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076236.960506:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587264:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076236.960560:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076236.960590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff8b1006135180 x1855007612587264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076236.960617:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612587264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076236.960635:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076236.960683:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076236.960692:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587264:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076236.960706:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076236.960749:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076236.960773:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff8b1006136a00 x1855007612587392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076236.960796:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612587392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076236.960812:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076236.960818:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076236.960824:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076236.960836:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076236.960877:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076236.960901:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff8b1006136300 x1855007612587520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076236.960926:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612587520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076236.960944:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076236.960952:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076236.960959:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076237.855793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076237.855885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076238.879833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076238.879869:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076239.903770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076239.903895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076240.927750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076240.927852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076241.951800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076241.951864:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076242.079907:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076242.079939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076242.079945:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076242.079957:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3230 00000100:00080000:0.0:1769076242.079972:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076242.079981:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076242.080055:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612587648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076242.080099:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076242.080104:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076242.080154:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076242.080160:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3230 00000100:00080000:0.0:1769076242.080166:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076242.080174:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076242.080174:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612587648:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076242.080207:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076242.080236:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612587776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076242.080256:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076242.080263:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076242.080267:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076242.080273:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3230 00000100:00080000:0.0:1769076242.080293:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076242.080295:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076242.080305:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076242.080320:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076242.080351:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff8b1006026300 x1855007612587648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076242.080375:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612587904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076242.080394:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076242.080394:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612587648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076242.080416:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076242.080420:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076242.080446:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3230 00000100:00080000:0.0:1769076242.080454:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076242.080460:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076242.080468:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076242.080475:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076242.080483:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076242.080486:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612588032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076242.080494:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612587648:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076242.080552:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587776:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076242.080580:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076242.080590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff8b1006024000 x1855007612587776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076242.080599:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612587776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076242.080605:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076242.080608:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076242.080610:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076242.080615:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076242.080626:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076242.080633:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff8b1006025880 x1855007612587904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076242.080676:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612587904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076242.080682:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076242.080685:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076242.080687:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612587904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076242.080692:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076242.080714:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076242.080722:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff8b1006026a00 x1855007612588032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076242.080730:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612588032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076242.080736:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076242.080738:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076242.080740:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076242.975714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076242.975777:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076243.999733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076243.999838:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076245.023701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076245.023954:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076246.047915:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076246.047973:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076247.071772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076247.071859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076247.199815:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.199834:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076247.199841:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.199857:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3235 00000100:00080000:0.0:1769076247.199876:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.199902:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076247.199962:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612588160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.200046:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.200054:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076247.200057:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.200074:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3235 00000100:00080000:0.0:1769076247.200080:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.200086:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076247.200103:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076247.200128:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612588288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076247.200146:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612588160:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076247.200150:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.200163:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076247.200166:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.200175:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3235 00000100:00080000:0.0:1769076247.200181:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.200190:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076247.200200:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076247.200221:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612588416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.200234:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.200238:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076247.200241:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.200246:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3235 00000100:00080000:0.0:1769076247.200261:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.200266:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076247.200271:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076247.200286:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612588544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076247.200729:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076247.200825:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076247] req@ffff8b1006027800 x1855007612588160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076247.200932:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612588160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076247.200956:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076247.200964:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076247.200973:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612588160:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076247.201041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588288:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076247.201096:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076247.201117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076247] req@ffff8b1006027100 x1855007612588288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076247.201144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612588288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076247.201157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076247.201161:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076247.201166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588288:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076247.201175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076247.201200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076247.201213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076247] req@ffff8b1006026680 x1855007612588416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076247.201229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612588416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076247.201239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076247.201243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076247.201247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076247.201254:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076247.201295:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076247.201317:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076247] req@ffff8b1006025180 x1855007612588544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076247.201343:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612588544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076247.201378:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076247.201386:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076247.201393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076248.095756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076248.095921:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076249.119727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076249.119952:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076250.143790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076250.143853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076251.167780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076251.167904:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076252.191771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076252.191895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076252.319756:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076252.319767:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076252.319772:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076252.319781:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3240 00000100:00080000:2.0:1769076252.319792:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076252.319799:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076252.319852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612588672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076252.319877:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076252.319880:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076252.319882:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076252.319885:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3240 00000100:00080000:2.0:1769076252.319888:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076252.319892:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076252.319899:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076252.319927:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612588800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076252.319954:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076252.319957:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076252.319959:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076252.319964:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3240 00000100:00080000:2.0:1769076252.319967:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076252.319971:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076252.319976:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076252.319988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612588672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076252.319989:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612588928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076252.320006:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076252.320008:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076252.320010:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076252.320012:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3240 00000100:00080000:2.0:1769076252.320016:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076252.320019:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076252.320023:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076252.320031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612589056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076252.320069:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076252.320105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff8b1005d5b480 x1855007612588672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.320124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612588672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.320137:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.320143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076252.320148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612588672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076252.320157:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588800:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076252.320186:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076252.320204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff8b1005d5a300 x1855007612588800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.320221:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612588800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.320232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.320236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076252.320242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588800:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076252.320272:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076252.320332:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076252.320350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff8b1005d58e00 x1855007612588928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.320368:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612588928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.320380:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.320384:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076252.320389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612588928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076252.320397:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076252.320438:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076252.320455:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff8b1005d5bb80 x1855007612589056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.320473:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612589056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.320484:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.320488:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076252.320493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076253.215914:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076253.216079:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076254.239813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076254.239977:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076255.263757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076255.263946:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076256.287770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076256.287938:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076257.311845:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076257.311957:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076257.439921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076257.439934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076257.439941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076257.439953:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3245 00000100:00080000:1.0:1769076257.439967:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076257.439975:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076257.440022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612589184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076257.440060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076257.440066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076257.440068:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076257.440073:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3245 00000100:00080000:1.0:1769076257.440077:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076257.440082:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076257.440107:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076257.440134:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612589312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076257.440143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612589184:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076257.440192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076257.440202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076257.440208:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076257.440218:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3245 00000100:00080000:1.0:1769076257.440229:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076257.440239:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769076257.440251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769076257.440252:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076257.440283:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612589440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076257.440305:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff8b1006135c00 x1855007612589184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.440315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076257.440330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076257.440337:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076257.440349:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3245 00000100:00080000:1.0:1769076257.440356:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076257.440367:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076257.440369:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612589184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076257.440375:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076257.440399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612589568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076257.440407:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076257.440424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076257.440435:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612589184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076257.440483:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589312:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076257.440521:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076257.440539:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff8b1006137b80 x1855007612589312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076257.440556:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612589312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076257.440567:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076257.440582:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076257.440586:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589312:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076257.440594:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076257.440616:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076257.440629:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff8b1006135880 x1855007612589440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076257.440684:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612589440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076257.440696:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076257.440700:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076257.440704:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076257.440712:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076257.440736:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076257.440751:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff8b1006135500 x1855007612589568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076257.440765:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612589568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076257.440775:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076257.440779:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076257.440782:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076258.335779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076258.335911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076259.359800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076259.359910:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076260.383845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076260.383871:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076261.407721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076261.407819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076262.431708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076262.431839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076262.559829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.559839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076262.559843:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.559851:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3250 00000100:00080000:3.0:1769076262.559861:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.559867:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076262.559900:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612589696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.559925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.559930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076262.559932:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.559936:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3250 00000100:00080000:3.0:1769076262.559940:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.559945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076262.559955:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076262.559969:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612589824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076262.559970:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612589696:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076262.559981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.559984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076262.559986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.559990:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3250 00000100:00080000:3.0:1769076262.559994:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.559998:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076262.560003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076262.560012:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612589952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.560031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.560050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076262.560052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.560055:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3250 00000100:00080000:3.0:1769076262.560058:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076262.560061:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076262.560062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076262.560066:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076262.560074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612590080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076262.560121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff8b10037ddc00 x1855007612589696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076262.560142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612589696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076262.560165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076262.560170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076262.560175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612589696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076262.560200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589824:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076262.560234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076262.560247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff8b10037dce00 x1855007612589824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076262.560257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612589824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076262.560263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076262.560266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076262.560268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589824:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076262.560273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076262.560288:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076262.560296:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff8b10037df480 x1855007612589952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076262.560326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612589952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076262.560336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076262.560339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076262.560360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612589952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076262.560369:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076262.560406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076262.560414:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff8b10037df800 x1855007612590080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076262.560422:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612590080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076262.560428:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076262.560430:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076262.560432:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076263.455743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076263.455900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076264.479723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076264.479959:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076265.503833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076265.504023:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076266.527808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076266.527938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076267.551882:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076267.551918:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076267.679827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076267.679840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076267.679844:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076267.679855:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3255 00000100:00080000:1.0:1769076267.679868:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076267.679876:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076267.679921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612590208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076267.679956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076267.679961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076267.679963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076267.679967:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3255 00000100:00080000:1.0:1769076267.679971:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076267.679975:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076267.679996:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076267.680013:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612590336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076267.680026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076267.680030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076267.680033:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076267.680038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3255 00000100:00080000:1.0:1769076267.680042:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076267.680050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612590208:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076267.680051:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076267.680066:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076267.680090:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612590464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076267.680108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076267.680115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076267.680119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076267.680125:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3255 00000100:00080000:1.0:1769076267.680146:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076267.680159:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769076267.680165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769076267.680169:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076267.680196:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612590592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076267.680218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff8b1006137100 x1855007612590208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076267.680242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612590208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076267.680257:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076267.680270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076267.680275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612590208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076267.680284:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590336:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076267.680310:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076267.680324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff8b1006134700 x1855007612590336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076267.680338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612590336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076267.680347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076267.680351:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076267.680355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590336:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076267.680381:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076267.680401:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076267.680411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff8b1006136d80 x1855007612590464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076267.680425:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612590464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076267.680434:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076267.680437:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076267.680440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076267.680446:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076267.680463:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076267.680473:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff8b1006134e00 x1855007612590592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076267.680486:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612590592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076267.680494:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076267.680497:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076267.680501:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590592:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769076268.575888:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076268.575973:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076269.599769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076269.599872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076270.623873:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076270.623900:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076271.647776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076271.647824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076272.671720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076272.671794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076272.799780:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076272.799791:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076272.799796:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076272.799806:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3260 00000100:00080000:0.0:1769076272.799818:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076272.799825:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076272.799872:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612590720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076272.799916:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076272.799923:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076272.799925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076272.799930:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3260 00000100:00080000:0.0:1769076272.799934:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076272.799939:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076272.799950:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076272.799966:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612590848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076272.799982:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076272.799982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612590720:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076272.799995:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076272.799999:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076272.800004:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3260 00000100:00080000:0.0:1769076272.800008:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076272.800011:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076272.800017:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076272.800030:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612590976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076272.800040:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076272.800043:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076272.800045:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076272.800049:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3260 00000100:00080000:0.0:1769076272.800056:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076272.800059:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076272.800064:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076272.800069:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076272.800081:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612591104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076272.800115:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff8b1006025180 x1855007612590720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076272.800138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612590720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076272.800149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076272.800154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076272.800158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612590720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076272.800181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590848:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076272.800203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076272.800214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff8b1006026680 x1855007612590848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076272.800226:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612590848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076272.800234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076272.800237:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076272.800241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076272.800260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076272.800279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076272.800289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff8b1006027100 x1855007612590976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076272.800301:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612590976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076272.800316:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076272.800319:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076272.800322:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612590976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076272.800327:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076272.800342:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076272.800350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff8b1006027800 x1855007612591104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076272.800361:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612591104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076272.800369:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076272.800372:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076272.800375:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076273.695770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076273.695840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076274.719776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076274.719864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076275.743772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076275.743970:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076276.767753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076276.767829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076277.791779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076277.791948:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076277.919801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076277.919812:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076277.919818:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076277.919829:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3266 00000100:00080000:1.0:1769076277.919841:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076277.919848:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076277.919888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612591232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076277.919913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076277.919917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076277.919919:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076277.919923:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3266 00000100:00080000:1.0:1769076277.919926:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076277.919930:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076277.919938:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076277.919950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612591360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076277.919961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076277.919964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076277.919966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076277.919973:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612591232:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076277.919974:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3266 00000100:00080000:1.0:1769076277.919982:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076277.919988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076277.919993:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076277.920005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612591488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076277.920020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076277.920023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076277.920025:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076277.920029:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3266 00000100:00080000:1.0:1769076277.920032:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076277.920035:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076277.920039:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076277.920050:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612591616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076277.920063:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076277.920111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff8b1006137480 x1855007612591232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076277.920143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612591232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076277.920155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076277.920160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076277.920164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612591232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076277.920187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591360:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076277.920212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076277.920224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff8b1006134a80 x1855007612591360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076277.920236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612591360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076277.920244:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076277.920247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076277.920250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591360:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076277.920255:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076277.920271:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076277.920280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff8b1006136680 x1855007612591488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076277.920290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612591488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076277.920298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076277.920300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076277.920303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076277.920309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076277.920323:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076277.920340:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff8b1006134e00 x1855007612591616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076277.920351:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612591616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076277.920358:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076277.920360:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076277.920363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076278.815773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076278.815838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076279.839786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076279.839816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076280.863783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076280.863927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076281.887780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076281.887803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076282.911793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076282.911883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076283.039775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076283.039790:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076283.039795:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076283.039810:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3271 00000100:00080000:3.0:1769076283.039828:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076283.039838:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076283.039888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612591744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076283.039930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076283.039935:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076283.039938:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076283.039946:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3271 00000100:00080000:3.0:1769076283.039952:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076283.039958:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076283.039974:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612591744:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076283.039974:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076283.040042:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076283.040071:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff8b10037dc700 x1855007612591744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076283.040093:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612591744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076283.040102:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076283.040106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076283.040109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612591744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076283.040237:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612591872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076283.040308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076283.040313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076283.040317:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076283.040324:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3271 00000100:00080000:3.0:1769076283.040330:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076283.040333:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591872:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769076283.040341:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076283.040353:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076283.040373:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076283.040375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612592000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076283.040392:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff8b10037df800 x1855007612591872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076283.040396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076283.040414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076283.040436:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076283.040440:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612591872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076283.040448:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3271 00000100:00080000:1.0:1769076283.040464:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076283.040464:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076283.040477:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00080000:3.0:1769076283.040482:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076283.040486:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612591872:192.168.204.102@tcp:38:kworker.0 00000008:00100000:3.0:1769076283.040497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076283.040509:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076283.040520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612592128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076283.040534:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076283.040549:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff8b10037df480 x1855007612592000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076283.040560:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612592000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076283.040566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076283.040569:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076283.040572:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076283.040580:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076283.040591:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076283.040597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076283/real 1769076283] req@ffff8b10037dce00 x1855007612592128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076283.040605:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612592128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076283.040610:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076283.040612:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076283.040614:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076283.935741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076283.935949:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076284.959817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076284.959919:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076285.983752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076285.983904:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076287.007793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076287.007897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076288.031775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076288.031936:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076288.159808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076288.159822:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076288.159828:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076288.159840:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3276 00000100:00080000:3.0:1769076288.159854:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076288.159863:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076288.159913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612592256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076288.159954:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076288.159970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076288.159973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076288.159979:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3276 00000100:00080000:3.0:1769076288.159983:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076288.159988:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076288.160000:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076288.160017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612592384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076288.160034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076288.160039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076288.160043:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076288.160050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612592256:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076288.160051:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3276 00000100:00080000:3.0:1769076288.160061:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076288.160071:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076288.160078:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076288.160094:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612592512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076288.160106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076288.160111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076288.160113:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076288.160118:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3276 00000100:00080000:3.0:1769076288.160124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076288.160128:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076288.160133:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076288.160146:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612592640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076288.160162:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076288.160229:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff8b10037ddc00 x1855007612592256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.160266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612592256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.160293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.160299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076288.160305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612592256:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076288.160366:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592384:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076288.160403:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076288.160422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff8b10037df100 x1855007612592384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.160440:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612592384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.160451:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.160455:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076288.160460:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076288.160468:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076288.160492:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076288.160505:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff8b10037dd880 x1855007612592512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.160520:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612592512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.160530:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.160534:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076288.160538:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076288.160546:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076288.160567:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076288.160579:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff8b10037dea00 x1855007612592640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.160593:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612592640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.160604:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.160607:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076288.160611:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076289.055857:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076289.055895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076290.079788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076290.079918:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076291.103725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076291.103790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076292.127737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076292.127802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076293.151732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076293.151881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076293.279773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076293.279783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076293.279789:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076293.279798:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3281 00000100:00080000:2.0:1769076293.279809:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076293.279815:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076293.279851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612592768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076293.279880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076293.279884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076293.279885:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076293.279889:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3281 00000100:00080000:2.0:1769076293.279892:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076293.279894:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076293.279902:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076293.279914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612592896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076293.279922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076293.279925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076293.279927:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076293.279929:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3281 00000100:00080000:2.0:1769076293.279931:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076293.279934:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076293.279937:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076293.279945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612593024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076293.279958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076293.279961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076293.279963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076293.279966:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3281 00000100:00080000:2.0:1769076293.279969:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076293.279971:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076293.279975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076293.279980:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612592768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076293.279987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612593152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076293.280047:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076293.280082:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff8b1005d59880 x1855007612592768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.280099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612592768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.280108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.280112:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076293.280116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612592768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076293.280122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592896:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076293.280139:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076293.280148:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff8b1005d59500 x1855007612592896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.280159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612592896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.280166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.280168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076293.280171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612592896:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076293.280175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076293.280189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076293.280197:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff8b1005d5aa00 x1855007612593024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.280206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612593024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.280212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.280215:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076293.280217:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076293.280239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076293.280276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076293.280285:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff8b1005d5b100 x1855007612593152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.280316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612593152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.280333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.280336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076293.280340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076294.175792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076294.175939:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076295.199740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076295.199821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076296.223862:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076296.223943:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076297.247767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076297.247868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076298.271784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076298.271895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076298.399762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076298.399777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076298.399782:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076298.399794:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3286 00000100:00080000:2.0:1769076298.399807:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076298.399816:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076298.399864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612593280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076298.399935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076298.399939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076298.399942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076298.399946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3286 00000100:00080000:2.0:1769076298.399949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076298.399953:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076298.399964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076298.399981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612593408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076298.399995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076298.400000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076298.400003:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076298.400008:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3286 00000100:00080000:2.0:1769076298.400014:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076298.400021:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076298.400046:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076298.400073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612593536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076298.400093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612593280:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076298.400097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076298.400111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076298.400119:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076298.400126:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3286 00000100:00080000:2.0:1769076298.400132:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076298.400137:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076298.400145:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076298.400186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612593664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076298.400290:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076298.400348:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076298.400373:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076299.295751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076299.295853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076300.319750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076300.319843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076301.343746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076301.343849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076301.472007:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076301.472062:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076301.472084:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076301.472154:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076301.472172:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076301.472188:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076301.472221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076301] req@ffff8b1005d5b100 x1855007612593536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076301.472265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612593536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076301.472305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076301.472313:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076301.472321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076301.472335:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076301] req@ffff8b1005d5aa00 x1855007612593664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076301.472363:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612593664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076301.472380:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076301.472387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076301.472394:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076301.472406:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076301] req@ffff8b1005d58380 x1855007612593280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076301.472431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612593280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076301.472447:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076301.472453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076301.472460:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612593280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076301.472473:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076301] req@ffff8b1005d5ad80 x1855007612593408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076301.472499:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612593408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076301.472516:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076301.472523:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076301.472530:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076302.367761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076302.367934:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076303.391828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076303.392204:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076303.519802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076303.519814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076303.519819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076303.519831:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3291 00000100:00080000:2.0:1769076303.519843:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076303.519851:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076303.519911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612593792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076303.519947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076303.519952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076303.519954:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076303.519960:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3291 00000100:00080000:2.0:1769076303.519965:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076303.519971:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076303.519983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076303.520002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612593920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076303.520016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076303.520023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076303.520025:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076303.520030:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3291 00000100:00080000:2.0:1769076303.520034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076303.520039:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076303.520044:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076303.520060:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612593792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076303.520064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612594048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076303.520144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076303.520151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076303.520156:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076303.520167:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3291 00000100:00080000:2.0:1769076303.520177:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076303.520201:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076303.520219:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076303.520235:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076303.520253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612594176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076303.520318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff8b1005d59500 x1855007612593792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.520364:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612593792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.520405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.520416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076303.520425:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612593792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076303.520443:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593920:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076303.520497:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076303.520528:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff8b1005d59880 x1855007612593920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.520586:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612593920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.520603:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.520611:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076303.520618:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612593920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076303.520750:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076303.520800:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076303.520826:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff8b1005d5bb80 x1855007612594048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.520851:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612594048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.520868:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.520875:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076303.520882:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076303.520898:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076303.520943:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076303.520969:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff8b1005d58e00 x1855007612594176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.520996:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612594176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.521013:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.521021:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076303.521028:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076304.415780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076304.415915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076305.439766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076305.440004:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076306.463854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076306.463944:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076307.487825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076307.487881:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076308.511877:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076308.511927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076308.639871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.639884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076308.639891:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.639903:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3296 00000100:00080000:3.0:1769076308.639917:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.639926:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076308.639974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612594304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.640044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.640049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076308.640052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.640058:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3296 00000100:00080000:3.0:1769076308.640063:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.640069:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076308.640081:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076308.640099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612594432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.640122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.640127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076308.640130:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.640135:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3296 00000100:00080000:3.0:1769076308.640139:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.640144:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076308.640153:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076308.640161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612594304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076308.640174:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612594560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.640191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.640196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076308.640199:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.640203:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3296 00000100:00080000:3.0:1769076308.640207:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.640212:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076308.640218:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076308.640232:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612594688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076308.640260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076308.640320:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff8b10037dc000 x1855007612594304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.640387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612594304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.640409:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.640418:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076308.640426:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612594304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076308.640437:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594432:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076308.640473:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076308.640491:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff8b10037dd500 x1855007612594432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.640509:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612594432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.640521:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.640525:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076308.640529:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076308.640568:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076308.640609:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076308.640634:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff8b10037ddf80 x1855007612594560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.640713:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612594560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.640724:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.640729:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076308.640733:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076308.640741:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076308.640772:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076308.640787:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff8b10037de300 x1855007612594688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.640801:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612594688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.640820:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.640824:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076308.640828:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076309.535847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076309.535934:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076310.559793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076310.559859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076311.583795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076311.583906:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076312.607861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076312.607870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076313.631799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076313.631882:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076313.759959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076313.759977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076313.759983:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076313.760015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3301 00000100:00080000:2.0:1769076313.760034:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076313.760045:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076313.760128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612594816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076313.760174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076313.760179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076313.760182:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076313.760188:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3301 00000100:00080000:2.0:1769076313.760192:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076313.760197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076313.760212:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076313.760231:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612594944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076313.760257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076313.760271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769076313.760272:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612594816:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076313.760277:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076313.760295:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3301 00000100:00080000:2.0:1769076313.760305:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076313.760315:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076313.760349:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076313.760378:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612595072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076313.760399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076313.760405:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076313.760409:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769076313.760417:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076313.760421:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3301 00000100:00080000:2.0:1769076313.760433:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076313.760441:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076313.760452:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076313.760482:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612595200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076313.760484:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076313] req@ffff8b1005d5a300 x1855007612594816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076313.760549:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612594816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076313.760564:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076313.760569:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076313.760578:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612594816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076313.760607:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594944:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076313.760691:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076313.760719:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076313] req@ffff8b1005d5b480 x1855007612594944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076313.760733:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612594944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076313.760742:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076313.760746:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076313.760749:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612594944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076313.760757:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076313.760778:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076313.760789:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076313] req@ffff8b1005d58000 x1855007612595072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076313.760801:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612595072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076313.760809:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076313.760829:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076313.760834:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076313.760841:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076313.760885:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076313.760897:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076313] req@ffff8b1005d59180 x1855007612595200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076313.760950:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612595200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076313.760959:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076313.760962:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076313.760965:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076314.655874:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076314.655968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076315.679825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076315.679912:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076316.703829:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076316.703978:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076317.727914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076317.727960:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076318.751795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076318.751920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076318.879870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.879894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076318.879903:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.879925:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3306 00000100:00080000:0.0:1769076318.879947:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.880145:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076318.880213:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612595328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.880259:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.880267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076318.880272:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.880278:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3306 00000100:00080000:0.0:1769076318.880291:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.880309:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076318.880325:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076318.880354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612595456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.880381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076318.880388:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612595328:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076318.880393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076318.880403:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.880417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3306 00000100:00080000:0.0:1769076318.880427:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.880436:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076318.880446:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076318.880468:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612595584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.880482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076318.880492:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076318.880494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076318.880501:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.880509:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3306 00000100:00080000:0.0:1769076318.880517:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.880523:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076318.880531:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076318.880548:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612595712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076318.880567:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff8b1006026a00 x1855007612595328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076318.880614:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612595328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076318.880636:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076318.880702:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076318.880710:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612595328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076318.880757:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076318.880802:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076318.880863:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff8b1006025880 x1855007612595456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076318.880887:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612595456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076318.880901:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076318.880907:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076318.880913:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076318.880923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076318.880952:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076318.880966:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff8b1006024000 x1855007612595584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076318.880981:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612595584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076318.880999:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076318.881003:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076318.881007:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076318.881014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076318.881033:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076318.881045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff8b1006026300 x1855007612595712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076318.881059:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612595712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076318.881069:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076318.881073:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076318.881076:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076319.775887:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076319.775898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076320.799869:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076320.799927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076321.823827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076321.823897:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076322.847740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076322.847829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076323.871790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076323.871934:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076323.999829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076323.999845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076323.999853:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076323.999869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3312 00000100:00080000:1.0:1769076323.999887:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076323.999898:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076323.999960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612595840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076324.000006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076324.000013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076324.000016:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076324.000021:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3312 00000100:00080000:1.0:1769076324.000027:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076324.000033:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076324.000048:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076324.000066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612595968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076324.000078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612595840:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076324.000088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076324.000096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076324.000098:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076324.000105:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3312 00000100:00080000:1.0:1769076324.000109:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076324.000114:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076324.000122:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076324.000138:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612596096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076324.000150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076324.000156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076324.000158:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076324.000163:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3312 00000100:00080000:1.0:1769076324.000181:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076324.000186:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076324.000194:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076324.000201:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076324.000218:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612596224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076324.000267:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076324] req@ffff8b1006136d80 x1855007612595840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.000297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612595840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.000313:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.000318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076324.000325:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612595840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076324.000361:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595968:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076324.000406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076324.000432:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076324] req@ffff8b1006134700 x1855007612595968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.000461:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612595968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.000480:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.000487:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076324.000495:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612595968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076324.000505:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076324.000539:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076324.000555:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076324] req@ffff8b1006137100 x1855007612596096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.000572:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612596096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.000582:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.000587:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076324.000591:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076324.000598:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076324.000619:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076324.000631:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076324] req@ffff8b1006135500 x1855007612596224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076324.000683:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612596224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076324.000694:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076324.000699:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076324.000703:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076324.895967:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076324.895979:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076325.919827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076325.919893:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076326.943792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076326.943901:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076327.967811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076327.967919:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076328.991881:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076328.991980:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076329.119841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076329.119853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076329.119858:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076329.119870:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3317 00000100:00080000:3.0:1769076329.119883:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076329.119891:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076329.119937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612596352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076329.119975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076329.119979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076329.119982:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076329.119987:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3317 00000100:00080000:3.0:1769076329.119991:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076329.119995:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076329.120006:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076329.120022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612596480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076329.120053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076329.120058:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076329.120063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076329.120076:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3317 00000100:00100000:0.0:1769076329.120081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612596352:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076329.120082:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076329.120095:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076329.120105:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076329.120125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612596608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076329.120139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076329.120144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076329.120147:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076329.120152:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3317 00000100:00080000:3.0:1769076329.120157:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076329.120161:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076329.120168:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076329.120185:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076329.120187:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612596736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076329.120250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff8b10037dd180 x1855007612596352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076329.120279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612596352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.120299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076329.120308:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076329.120317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612596352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076329.120332:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076329.120393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076329.120416:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff8b10037ded80 x1855007612596480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076329.120433:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612596480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.120443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076329.120447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076329.120452:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076329.120483:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076329.120509:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076329.120524:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff8b10037de680 x1855007612596608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076329.120538:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612596608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.120548:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076329.120551:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076329.120555:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076329.120562:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076329.120579:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076329.120590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff8b10037dc380 x1855007612596736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076329.120604:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612596736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076329.120613:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076329.120617:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076329.120620:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076330.015764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076330.015921:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076331.039784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076331.039877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076332.063788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076332.063820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076333.087809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076333.087939:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076334.111735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076334.111832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076334.239833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076334.239847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076334.239853:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076334.239865:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3322 00000100:00080000:1.0:1769076334.239880:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076334.239889:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076334.239935:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612596864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076334.239982:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076334.239987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076334.239989:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076334.239995:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3322 00000100:00080000:1.0:1769076334.239999:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076334.240004:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076334.240016:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076334.240036:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612596992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076334.240052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076334.240057:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076334.240060:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076334.240066:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3322 00000100:00080000:1.0:1769076334.240074:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076334.240086:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076334.240090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612596864:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076334.240095:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076334.240117:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612597120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076334.240137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076334.240141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076334.240145:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076334.240158:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3322 00000100:00080000:1.0:1769076334.240165:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076334.240175:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076334.240186:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076334.240212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612597248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076334.240257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076334.240324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff8b1006135880 x1855007612596864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076334.240383:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612596864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076334.240406:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076334.240416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076334.240425:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612596864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076334.240443:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076334.240492:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076334.240513:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff8b1006137b80 x1855007612596992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076334.240530:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612596992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076334.240542:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076334.240587:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076334.240593:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612596992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076334.240635:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076334.240712:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076334.240729:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff8b1006135c00 x1855007612597120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076334.240745:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612597120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076334.240765:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076334.240770:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076334.240774:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076334.240782:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076334.240805:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076334.240817:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff8b1006136300 x1855007612597248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076334.240832:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612597248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076334.240842:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076334.240846:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076334.240849:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076335.135779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076335.135905:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076336.159781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076336.159913:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076337.183786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076337.183858:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076338.207817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076338.207922:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076339.231859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076339.231880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076339.359822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.359840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076339.359847:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.359864:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3327 00000100:00080000:3.0:1769076339.359881:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.359894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076339.359952:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612597376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.360002:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.360010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076339.360014:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.360022:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3327 00000100:00080000:3.0:1769076339.360030:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.360038:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076339.360056:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076339.360075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612597376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076339.360083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612597504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.360103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.360108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076339.360112:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.360118:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3327 00000100:00080000:3.0:1769076339.360124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.360131:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076339.360139:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076339.360156:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612597632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076339.360168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076339.360175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.360180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076339.360184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.360190:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3327 00000100:00080000:3.0:1769076339.360196:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.360202:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076339.360211:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076339.360224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff8b10037dca80 x1855007612597376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076339.360228:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612597760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076339.360259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612597376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076339.360291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076339.360299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076339.360308:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612597376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076339.360344:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597504:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076339.360381:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076339.360399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff8b10037dc380 x1855007612597504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076339.360415:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612597504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076339.360426:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076339.360430:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076339.360436:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076339.360447:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076339.360511:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076339.360536:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff8b10037de680 x1855007612597632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076339.360561:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612597632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076339.360577:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076339.360584:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076339.360591:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076339.360604:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076339.360697:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076339.360719:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff8b10037ded80 x1855007612597760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076339.360735:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612597760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076339.360746:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076339.360751:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076339.360755:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612597760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076340.255830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076340.255858:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076341.279784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076341.279936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076342.303786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076342.303902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076343.327829:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076343.327940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076344.351806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076344.352000:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076344.479858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076344.479871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076344.479877:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076344.479890:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3332 00000100:00080000:3.0:1769076344.479904:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076344.479915:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076344.479967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612597888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076344.480006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076344.480014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076344.480018:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076344.480027:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3332 00000100:00080000:3.0:1769076344.480034:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076344.480041:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076344.480056:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076344.480081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612598016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076344.480098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612597888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076344.480103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076344.480110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076344.480114:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076344.480120:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3332 00000100:00080000:3.0:1769076344.480127:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076344.480133:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076344.480142:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076344.480159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612598144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076344.480175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076344.480181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076344.480184:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076344.480189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076344.480192:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3332 00000100:00080000:3.0:1769076344.480198:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076344.480204:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076344.480213:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076344.480229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612598272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076344.480246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff8b10037dd180 x1855007612597888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076344.480280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612597888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076344.480300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076344.480309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076344.480317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612597888:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076344.480357:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598016:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076344.480429:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076344.480450:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff8b10037de300 x1855007612598016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076344.480466:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612598016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076344.480477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076344.480482:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076344.480486:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076344.480494:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076344.480519:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076344.480532:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff8b10037ddf80 x1855007612598144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076344.480546:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612598144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076344.480556:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076344.480560:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076344.480563:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076344.480570:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076344.480588:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076344.480600:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff8b10037dd500 x1855007612598272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076344.480615:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612598272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076344.480625:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076344.480628:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076344.480632:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076345.375801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076345.376004:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076346.399837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076346.399944:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076347.423765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076347.423895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076348.447769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076348.447944:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076349.471830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076349.471953:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076349.599764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.599780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076349.599788:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.599804:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3337 00000100:00080000:3.0:1769076349.599823:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.599835:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076349.599915:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612598400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.599963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.599969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076349.599972:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.599979:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3337 00000100:00080000:3.0:1769076349.599985:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.599992:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076349.600008:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076349.600025:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612598400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076349.600031:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612598528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.600053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.600058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076349.600061:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.600067:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3337 00000100:00080000:3.0:1769076349.600072:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.600077:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076349.600083:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076349.600100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612598656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.600122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.600127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076349.600129:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.600133:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3337 00000100:00080000:3.0:1769076349.600139:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.600144:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076349.600150:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076349.600160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076349.600169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612598784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076349.600234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff8b10037dc000 x1855007612598400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076349.600261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612598400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076349.600277:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076349.600283:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076349.600289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612598400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076349.600352:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598528:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076349.600387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076349.600405:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff8b10037dea00 x1855007612598528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076349.600431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612598528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076349.600443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076349.600447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076349.600452:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076349.600460:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076349.600482:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076349.600494:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff8b10037dd880 x1855007612598656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076349.600510:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612598656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076349.600521:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076349.600524:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076349.600528:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076349.600535:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076349.600556:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076349.600567:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff8b10037df100 x1855007612598784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076349.600582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612598784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076349.600601:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076349.600608:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076349.600615:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612598784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076350.495756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076350.495953:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076351.519800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076351.519846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076352.543745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076352.543822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076353.567843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076353.567879:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076354.591763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076354.591913:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076354.719750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.719763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076354.719770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.719781:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3342 00000100:00080000:3.0:1769076354.719797:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.719807:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076354.719855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612598912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.719910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.719916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076354.719919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.719928:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3342 00000100:00080000:3.0:1769076354.719934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.719941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076354.719955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076354.719976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612599040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076354.719987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612598912:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076354.719997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.720006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076354.720009:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.720016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3342 00000100:00080000:3.0:1769076354.720020:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.720025:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076354.720032:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076354.720051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612599168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.720065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.720070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076354.720073:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.720080:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3342 00000100:00080000:3.0:1769076354.720087:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.720092:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076354.720099:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076354.720111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076354.720120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612599296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076354.720182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff8b10037ddc00 x1855007612598912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076354.720247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612598912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076354.720269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076354.720276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076354.720282:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612598912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076354.720316:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076354.720353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076354.720370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff8b10037dce00 x1855007612599040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076354.720387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612599040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076354.720398:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076354.720402:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076354.720407:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076354.720414:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076354.720450:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076354.720467:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff8b10037df480 x1855007612599168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076354.720483:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612599168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076354.720494:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076354.720498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076354.720502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076354.720509:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076354.720533:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076354.720546:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff8b10037df800 x1855007612599296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076354.720561:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612599296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076354.720571:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076354.720574:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076354.720578:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076355.615767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076355.615950:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076356.639734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076356.639787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076357.663781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076357.663917:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076358.687828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076358.687953:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076359.711792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076359.711939:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076359.839779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.839793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076359.839799:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.839811:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3347 00000100:00080000:3.0:1769076359.839837:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.839845:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076359.839893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612599424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.839942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.839947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076359.839949:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.839955:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3347 00000100:00080000:3.0:1769076359.839959:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.839964:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076359.839976:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076359.839994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612599552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076359.840007:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612599424:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076359.840015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.840022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076359.840024:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.840032:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3347 00000100:00080000:3.0:1769076359.840037:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.840042:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076359.840048:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076359.840065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612599680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.840078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.840083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076359.840085:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.840089:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3347 00000100:00080000:3.0:1769076359.840093:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.840102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769076359.840106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769076359.840111:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076359.840127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612599808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076359.840198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff8b10037dc700 x1855007612599424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076359.840226:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612599424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076359.840240:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076359.840246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076359.840252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612599424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076359.840279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076359.840309:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076359.840324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff8b10037df800 x1855007612599552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076359.840340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612599552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076359.840351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076359.840355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076359.840359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076359.840366:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076359.840387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076359.840398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff8b10037df480 x1855007612599680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076359.840423:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612599680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076359.840442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076359.840447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076359.840451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076359.840458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076359.840477:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076359.840488:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff8b10037dce00 x1855007612599808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076359.840503:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612599808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076359.840514:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076359.840517:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076359.840521:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612599808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076360.735861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076360.735895:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076361.759794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076361.759926:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076362.783768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076362.783936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076363.807721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076363.807796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076364.831775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076364.831907:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076364.959845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076364.959857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076364.959862:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076364.959887:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3353 00000100:00080000:2.0:1769076364.959901:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076364.959911:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076364.959959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612599936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076364.960001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076364.960006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076364.960009:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076364.960014:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3353 00000100:00080000:2.0:1769076364.960018:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076364.960023:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076364.960034:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076364.960090:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612600064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076364.960106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076364.960112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076364.960117:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076364.960126:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3353 00000100:00080000:2.0:1769076364.960134:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076364.960143:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076364.960151:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612599936:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769076364.960174:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076364.960209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612600192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076364.960234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076364.960242:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076364.960245:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076364.960251:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3353 00000100:00080000:2.0:1769076364.960258:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076364.960263:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076364.960272:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076364.960291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612600320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076364.960302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076364.960361:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076364.960391:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076365.855798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076365.855884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076366.879757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076366.879855:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076367.903890:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076367.903950:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076368.032137:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076368.032205:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076368.032238:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076368.032311:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076368.032335:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076368.032351:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769076368.032442:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076368] req@ffff8b1005d59180 x1855007612600192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076368.032504:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612600192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076368.032526:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076368.032537:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076368.032546:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076368.032562:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076368] req@ffff8b1005d58000 x1855007612600320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076368.032590:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612600320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076368.032607:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076368.032614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076368.032622:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076368.032635:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076368] req@ffff8b1005d5b800 x1855007612599936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076368.032718:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612599936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076368.032736:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076368.032743:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076368.032753:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612599936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076368.032769:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076368] req@ffff8b1005d58a80 x1855007612600064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076368.032793:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612600064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076368.032808:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076368.032815:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076368.032821:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076368.927761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076368.927914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076369.951755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076369.951908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076370.079821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.079837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076370.079847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.079865:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3358 00000100:00080000:3.0:1769076370.079883:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.079894:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076370.079951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612600448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076370.080039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.080045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076370.080048:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.080055:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3358 00000100:00080000:3.0:1769076370.080060:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.080067:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076370.080093:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076370.080114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612600576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076370.080131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.080138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076370.080143:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076370.080152:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612600448:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076370.080156:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3358 00000100:00080000:3.0:1769076370.080169:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.080180:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076370.080191:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076370.080245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612600704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076370.080278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.080285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076370.080288:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076370.080301:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076370.080307:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3358 00000100:00080000:3.0:1769076370.080321:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.080331:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076370.080344:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076370.080382:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612600832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076370.080398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff8b10037ddc00 x1855007612600448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.080447:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612600448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.080470:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.080481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076370.080491:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612600448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076370.080540:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076370.080624:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076370.080716:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff8b10037df100 x1855007612600576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.080746:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612600576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.080764:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.080772:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076370.080780:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076370.080794:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.080841:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076370.080865:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff8b10037dd880 x1855007612600704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.080905:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612600704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.080923:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.080930:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076370.080937:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076370.080951:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076370.081016:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076370.081043:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff8b10037dea00 x1855007612600832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076370.081069:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612600832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076370.081086:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076370.081093:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076370.081100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612600832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076370.975745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076370.975881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076371.999752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076371.999898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076373.023812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076373.023937:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076374.047787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076374.047886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076375.071869:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076375.071917:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076375.199763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.199776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076375.199782:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.199794:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3363 00000100:00080000:3.0:1769076375.199808:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.199817:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076375.199863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612600960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.199928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.199933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076375.199936:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.199944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3363 00000100:00080000:3.0:1769076375.199950:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.199956:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076375.199968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076375.199994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612601088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076375.200012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612600960:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076375.200025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.200035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076375.200040:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.200050:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3363 00000100:00080000:3.0:1769076375.200061:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.200071:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076375.200081:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076375.200107:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076375.200110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612601216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.200135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.200142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076375.200146:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.200156:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3363 00000100:00100000:0.0:1769076375.200161:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff8b10037dc000 x1855007612600960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076375.200169:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.200188:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076375.200202:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612600960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076375.200202:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076375.200228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076375.200240:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076375.200240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612601344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076375.200254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612600960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076375.200302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601088:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076375.200334:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076375.200351:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff8b10037dd500 x1855007612601088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076375.200367:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612601088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076375.200377:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076375.200382:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076375.200386:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076375.200394:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076375.200414:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076375.200426:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff8b10037ddf80 x1855007612601216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076375.200444:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612601216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076375.200461:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076375.200467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076375.200474:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076375.200486:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076375.200526:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076375.200549:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff8b10037de300 x1855007612601344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076375.200567:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612601344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076375.200578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076375.200582:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076375.200586:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076376.095711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076376.095815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076377.119820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076377.119935:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076378.143775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076378.143960:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076379.167775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076379.167968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076380.191764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076380.191917:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076380.319699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076380.319708:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076380.319713:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076380.319722:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3368 00000100:00080000:3.0:1769076380.319733:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076380.319739:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076380.319803:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612601472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076380.319858:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076380.319865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076380.319869:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076380.319877:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3368 00000100:00080000:3.0:1769076380.319884:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076380.319891:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076380.319903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076380.319923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612601600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076380.319940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612601472:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076380.319940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076380.319955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076380.319961:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076380.319966:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3368 00000100:00080000:3.0:1769076380.319970:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076380.319975:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076380.320000:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076380.320019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612601728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076380.320033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076380.320039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076380.320041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076380.320045:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3368 00000100:00080000:3.0:1769076380.320051:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076380.320059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076380.320062:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076380.320080:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076380.320105:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612601856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076380.320142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076380] req@ffff8b10037dd180 x1855007612601472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076380.320166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612601472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076380.320180:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076380.320187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076380.320194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612601472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076380.320222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601600:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076380.320247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076380.320261:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076380] req@ffff8b10037ded80 x1855007612601600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076380.320275:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612601600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076380.320284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076380.320287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076380.320291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076380.320297:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076380.320315:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076380.320325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076380] req@ffff8b10037de680 x1855007612601728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076380.320338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612601728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076380.320347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076380.320366:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076380.320369:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076380.320373:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076380.320387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076380.320395:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076380] req@ffff8b10037dc380 x1855007612601856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076380.320405:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612601856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076380.320412:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076380.320414:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076380.320417:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612601856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076381.215733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076381.215832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076382.239932:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076382.239996:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076383.263777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076383.263854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076384.287760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076384.288017:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076385.311777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076385.311902:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076385.439768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076385.439783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076385.439805:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076385.439819:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3373 00000100:00080000:3.0:1769076385.439837:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076385.439847:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076385.439901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612601984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076385.439936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076385.439941:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076385.439943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076385.439947:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3373 00000100:00080000:3.0:1769076385.439950:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076385.439954:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076385.439966:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076385.439999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612602112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076385.440012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076385.440016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076385.440020:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076385.440041:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3373 00000100:00080000:3.0:1769076385.440047:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076385.440049:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612601984:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076385.440056:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076385.440071:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076385.440091:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612602240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076385.440101:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076385.440105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076385.440108:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076385.440112:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3373 00000100:00080000:3.0:1769076385.440115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076385.440119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076385.440124:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076385.440135:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612602368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076385.440158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076385.440204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff8b10037dca80 x1855007612601984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076385.440228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612601984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076385.440242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076385.440248:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076385.440254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612601984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076385.440285:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076385.440327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076385.440350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff8b10037dc380 x1855007612602112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076385.440377:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612602112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076385.440407:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076385.440415:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076385.440422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076385.440435:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076385.440479:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076385.440503:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff8b10037de680 x1855007612602240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076385.440525:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612602240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076385.440536:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076385.440540:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076385.440554:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076385.440564:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076385.440602:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076385.440638:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff8b10037ded80 x1855007612602368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076385.440666:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612602368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076385.440745:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076385.440751:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076385.440757:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076386.335840:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076386.335948:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076387.359784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076387.359888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076388.383764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076388.383931:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076389.407913:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076389.407974:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076390.431737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076390.431946:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076390.559763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076390.559781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076390.559789:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076390.559806:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3378 00000100:00080000:3.0:1769076390.559824:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076390.559837:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076390.559902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612602496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076390.559955:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076390.559961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076390.559964:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076390.559970:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3378 00000100:00080000:3.0:1769076390.559978:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076390.559985:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076390.560001:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076390.560024:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612602496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076390.560027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612602624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076390.560060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076390.560067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076390.560072:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076390.560083:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3378 00000100:00080000:3.0:1769076390.560091:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076390.560096:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076390.560103:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076390.560128:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612602752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076390.560142:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076390.560147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076390.560149:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769076390.560157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076390.560160:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3378 00000100:00080000:3.0:1769076390.560185:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076390.560191:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076390.560198:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076390.560223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff8b10037dd180 x1855007612602496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076390.560224:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612602880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076390.560316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612602496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076390.560331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076390.560338:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076390.560343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612602496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076390.560379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602624:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076390.560415:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076390.560442:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff8b10037de300 x1855007612602624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076390.560463:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612602624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076390.560475:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076390.560481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076390.560486:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076390.560496:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076390.560519:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076390.560531:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff8b10037ddf80 x1855007612602752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076390.560549:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612602752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076390.560568:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076390.560572:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076390.560576:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076390.560592:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076390.560678:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076390.560697:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff8b10037dd500 x1855007612602880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076390.560718:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612602880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076390.560731:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076390.560736:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076390.560741:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612602880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076391.455775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076391.455867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076392.479887:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076392.479961:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076393.503749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076393.503904:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076394.527739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076394.527894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076395.551780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076395.551903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076395.679783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076395.679795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076395.679800:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076395.679813:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3383 00000100:00080000:2.0:1769076395.679828:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076395.679837:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076395.679885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612603008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076395.679927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076395.679933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076395.679935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076395.679940:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3383 00000100:00080000:2.0:1769076395.679945:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076395.679949:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076395.679961:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076395.679977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612603136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076395.679994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076395.679999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076395.680003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076395.680010:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612603008:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076395.680014:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3383 00000100:00080000:2.0:1769076395.680025:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076395.680031:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076395.680039:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076395.680061:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612603264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076395.680081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076395.680087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076395.680092:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076395.680100:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3383 00000800:00000100:1.0:1769076395.680111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076395.680113:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076395.680128:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076395.680139:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076395.680170:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612603392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076395.680173:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff8b1005d5b480 x1855007612603008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076395.680234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612603008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076395.680251:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076395.680264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076395.680278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612603008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076395.680323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076395.680369:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076395.680396:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff8b1005d5a300 x1855007612603136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076395.680421:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612603136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076395.680433:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076395.680438:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076395.680446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076395.680459:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076395.680504:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076395.680552:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff8b1005d58e00 x1855007612603264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076395.680572:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612603264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076395.680583:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076395.680587:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076395.680592:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076395.680600:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076395.680629:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076395.680682:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff8b1005d5bb80 x1855007612603392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076395.680699:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612603392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076395.680710:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076395.680713:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076395.680717:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076396.575779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076396.575809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076397.599786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076397.599894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076398.623756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076398.623870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076399.647786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076399.647880:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076400.671774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076400.671892:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076400.799901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076400.799918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076400.799925:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076400.799941:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3388 00000100:00080000:0.0:1769076400.799960:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076400.799971:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076400.800070:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612603520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076400.800141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076400.800149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076400.800154:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076400.800163:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3388 00000100:00080000:0.0:1769076400.800171:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076400.800181:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076400.800201:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076400.800248:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612603648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076400.800248:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612603520:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076400.800308:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076400.800313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076400.800317:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076400.800322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3388 00000100:00080000:0.0:1769076400.800328:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076400.800335:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076400.800344:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076400.800360:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612603776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076400.800373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076400.800378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076400.800380:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076400.800384:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3388 00000100:00080000:0.0:1769076400.800389:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076400.800393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076400.800400:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076400.800410:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076400.800426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612603904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076400.800501:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff8b1006026d80 x1855007612603520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.800552:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612603520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.800568:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.800575:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076400.800581:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612603520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076400.800619:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076400.800654:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076400.800749:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff8b1006027b80 x1855007612603648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.800775:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612603648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.800805:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.800812:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076400.800818:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076400.800830:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076400.800892:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076400.800917:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff8b1006025f80 x1855007612603776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.800967:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612603776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.800980:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.800984:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076400.800989:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076400.800997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076400.801059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076400.801074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff8b1006024a80 x1855007612603904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.801090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612603904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.801100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.801106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076400.801132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612603904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076401.695802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076401.695943:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076402.719776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076402.719952:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076403.743868:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076403.743942:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076404.767708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076404.767771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076405.791755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076405.791871:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076405.919860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076405.919872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076405.919878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076405.919889:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3394 00000100:00080000:2.0:1769076405.919903:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076405.919911:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076405.919955:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612604032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076405.919994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076405.919999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076405.920001:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076405.920005:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3394 00000100:00080000:2.0:1769076405.920009:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076405.920013:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076405.920023:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076405.920037:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612604160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076405.920049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076405.920053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076405.920056:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076405.920060:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3394 00000100:00080000:2.0:1769076405.920064:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076405.920068:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076405.920073:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076405.920082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612604032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076405.920090:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612604288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076405.920108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076405.920112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076405.920114:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076405.920117:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3394 00000100:00080000:2.0:1769076405.920123:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076405.920126:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076405.920131:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076405.920144:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612604416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076405.920183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076405.920226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff8b1005d59880 x1855007612604032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.920248:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612604032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.920261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.920266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076405.920271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612604032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076405.920280:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604160:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076405.920303:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076405.920316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff8b1005d59500 x1855007612604160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.920330:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612604160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.920340:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.920343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076405.920347:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076405.920420:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076405.920447:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076405.920461:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff8b1005d5ad80 x1855007612604288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.920480:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612604288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.920495:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.920501:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076405.920506:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076405.920517:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076405.920552:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076405.920572:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff8b1005d58380 x1855007612604416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.920591:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612604416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.920600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.920604:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076405.920607:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076406.815755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076406.815850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076407.839707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076407.839798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076408.863798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076408.863868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076409.887788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076409.887887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076410.911844:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076410.911856:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076411.039780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076411.039793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076411.039798:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076411.039810:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3399 00000100:00080000:2.0:1769076411.039825:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076411.039834:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076411.039881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612604544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076411.039920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076411.039925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076411.039928:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076411.039933:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3399 00000100:00080000:2.0:1769076411.039939:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076411.039944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076411.039955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076411.039971:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612604672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076411.039987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076411.039993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076411.039996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076411.040003:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612604544:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076411.040006:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3399 00000100:00080000:2.0:1769076411.040017:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076411.040025:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076411.040031:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076411.040049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612604800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076411.040061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076411.040066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076411.040069:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076411.040073:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3399 00000100:00080000:2.0:1769076411.040078:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076411.040082:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076411.040088:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076411.040102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076411.040105:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612604928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076411.040178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff8b1005d5aa00 x1855007612604544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.040206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612604544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076411.040220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076411.040227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076411.040232:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612604544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076411.040261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076411.040338:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076411.040357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff8b1005d5b100 x1855007612604672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.040374:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612604672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076411.040385:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076411.040389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076411.040394:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076411.040402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076411.040426:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076411.040438:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff8b1005d58380 x1855007612604800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.040453:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612604800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076411.040464:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076411.040467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076411.040471:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076411.040479:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076411.040499:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076411.040510:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076411/real 1769076411] req@ffff8b1005d5ad80 x1855007612604928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076411.040525:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612604928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076411.040535:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076411.040539:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076411.040543:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612604928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076411.935818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076411.935911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076412.959733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076412.959784:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076413.983794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076413.983950:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076415.007769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076415.007900:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076416.031803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076416.031926:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076416.159746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076416.159758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076416.159763:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076416.159774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3404 00000100:00080000:2.0:1769076416.159787:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076416.159795:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076416.159838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612605056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076416.159877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076416.159884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076416.159887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076416.159894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3404 00000100:00080000:2.0:1769076416.159900:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076416.159907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076416.159920:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076416.159941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612605184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076416.159958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076416.159964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076416.159968:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076416.159972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612605056:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076416.159973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3404 00000100:00080000:2.0:1769076416.159980:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076416.159985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076416.159993:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076416.160010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612605312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076416.160025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076416.160030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076416.160032:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076416.160038:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3404 00000100:00080000:2.0:1769076416.160042:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076416.160047:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769076416.160054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769076416.160054:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076416.160069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612605440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076416.160104:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff8b1005d59500 x1855007612605056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076416.160134:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612605056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076416.160152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076416.160157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076416.160162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612605056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076416.160173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605184:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076416.160254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076416.160277:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff8b1005d59880 x1855007612605184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076416.160300:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612605184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076416.160314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076416.160318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076416.160322:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076416.160351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076416.160377:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076416.160389:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff8b1005d5bb80 x1855007612605312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076416.160403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612605312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076416.160411:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076416.160414:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076416.160418:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076416.160424:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076416.160440:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076416.160449:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff8b1005d58e00 x1855007612605440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076416.160461:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612605440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076416.160476:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076416.160479:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076416.160482:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076417.055834:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076417.055916:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769076418.079873:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076418.079874:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076419.103750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076419.103901:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076420.127744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076420.127932:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076421.151788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076421.151901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076421.279757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.279768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076421.279773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.279785:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3409 00000100:00080000:2.0:1769076421.279800:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.279808:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076421.279856:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612605568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076421.279893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.279898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076421.279901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.279905:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3409 00000100:00080000:2.0:1769076421.279910:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.279914:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076421.279925:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076421.279941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612605696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076421.279957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076421.279967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612605568:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076421.279968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076421.279976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.279990:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3409 00000100:00080000:2.0:1769076421.279993:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.279997:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076421.280004:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076421.280020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612605824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076421.280033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.280038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076421.280041:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.280045:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3409 00000100:00080000:2.0:1769076421.280051:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.280055:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076421.280061:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076421.280070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076421.280085:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612605952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076421.280151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff8b1005d5a300 x1855007612605568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.280178:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612605568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.280192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.280198:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076421.280203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612605568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076421.280234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076421.280263:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076421.280279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff8b1005d5b480 x1855007612605696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.280295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612605696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.280306:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.280310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076421.280314:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076421.280323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076421.280344:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076421.280356:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff8b1005d58a80 x1855007612605824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.280371:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612605824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.280381:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.280385:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076421.280388:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076421.280395:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076421.280416:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076421.280427:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff8b1005d5b800 x1855007612605952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.280441:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612605952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.280451:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.280455:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076421.280458:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612605952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076422.175739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076422.175927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076423.199748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076423.199887:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076424.223758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076424.223884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076425.247864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076425.247912:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076426.271742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076426.271875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076426.399772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076426.399785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076426.399790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076426.399802:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3414 00000100:00080000:2.0:1769076426.399816:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076426.399826:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076426.399872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612606080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076426.399914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076426.399919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076426.399922:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076426.399927:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3414 00000100:00080000:2.0:1769076426.399932:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076426.399937:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076426.399947:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076426.399962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612606208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076426.399975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076426.399980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076426.399982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076426.399986:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3414 00000100:00080000:2.0:1769076426.399990:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076426.399994:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076426.400000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076426.400042:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612606336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076426.400055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076426.400060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076426.400063:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076426.400068:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3414 00000100:00080000:2.0:1769076426.400073:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076426.400078:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076426.400086:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076426.400088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612606080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076426.400104:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612606464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076426.400197:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076426.400252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff8b1005d58000 x1855007612606080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.400281:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612606080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.400298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.400303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076426.400309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612606080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076426.400319:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076426.400361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076426.400387:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff8b1005d59180 x1855007612606208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.400418:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612606208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.400435:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.400441:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076426.400448:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076426.400463:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076426.400518:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076426.400554:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff8b1005d5b800 x1855007612606336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.400573:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612606336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.400598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.400607:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076426.400615:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076426.400738:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076426.400773:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076426.400789:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff8b1005d58a80 x1855007612606464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.400805:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612606464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.400817:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.400821:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076426.400826:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076427.295721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076427.295773:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076428.319701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076428.319950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076429.343833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076429.343898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076430.367779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076430.367855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076431.391796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076431.392004:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076431.519788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076431.519799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076431.519804:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076431.519816:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3419 00000100:00080000:2.0:1769076431.519830:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076431.519839:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076431.519887:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612606592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076431.519922:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076431.519952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076431.519955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076431.519961:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3419 00000100:00080000:2.0:1769076431.519968:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076431.519974:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076431.519987:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076431.520009:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612606720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076431.520020:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612606592:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076431.520031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076431.520038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076431.520042:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076431.520049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3419 00000100:00080000:2.0:1769076431.520055:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076431.520060:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076431.520067:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076431.520084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612606848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076431.520104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076431.520111:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076431.520115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076431.520123:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3419 00000100:00080000:2.0:1769076431.520131:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076431.520138:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076431.520150:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076431.520177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076431.520181:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612606976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076431.520237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076431.520275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606976:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769076432.415828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076432.415841:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076433.439800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076433.439881:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076434.463856:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076434.463966:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769076434.592069:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769076434.592174:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076434.592196:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076434.592243:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076434.592255:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076434.592271:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076434.592330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076434] req@ffff8b1005d5bb80 x1855007612606976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076434.592396:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612606976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076434.592420:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076434.592430:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076434.592439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076434.592454:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076434] req@ffff8b1005d5a300 x1855007612606720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076434.592481:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612606720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076434.592497:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076434.592504:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076434.592510:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076434.592522:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076434] req@ffff8b1005d58e00 x1855007612606848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076434.592544:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612606848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076434.592561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076434.592568:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076434.592573:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612606848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076434.592586:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076434] req@ffff8b1005d5b480 x1855007612606592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076434.592608:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612606592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076434.592622:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076434.592628:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076434.592634:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612606592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076435.487782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076435.487900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076436.511765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076436.511911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076436.639766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.639780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076436.639786:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.639801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3424 00000100:00080000:2.0:1769076436.639820:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.639832:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076436.639909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612607104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.639947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.639952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076436.639955:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.639961:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3424 00000100:00080000:2.0:1769076436.639966:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.639972:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076436.639984:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076436.640004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612607232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.640019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.640024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076436.640031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076436.640033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612607104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076436.640041:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3424 00000100:00080000:2.0:1769076436.640053:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.640058:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076436.640064:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076436.640085:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612607360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.640099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.640105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076436.640109:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.640116:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3424 00000100:00080000:2.0:1769076436.640125:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076436.640135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076436.640138:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076436.640155:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076436.640182:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612607488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076436.640199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff8b1005d59880 x1855007612607104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076436.640259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612607104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076436.640279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076436.640287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076436.640296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612607104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076436.640337:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076436.640382:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076436.640422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff8b1005d59500 x1855007612607232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076436.640452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612607232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076436.640470:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076436.640477:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076436.640484:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076436.640498:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076436.640539:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076436.640564:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff8b1005d5ad80 x1855007612607360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076436.640590:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612607360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076436.640607:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076436.640615:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076436.640622:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076436.640689:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076436.640738:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076436.640762:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff8b1005d58380 x1855007612607488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076436.640789:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612607488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076436.640807:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076436.640814:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076436.640820:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076437.535737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076437.535963:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076438.559840:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076438.559932:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076439.583856:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076439.583861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076440.607753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076440.607833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076441.631755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076441.631922:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076441.759778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076441.759805:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076441.759809:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076441.759819:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3429 00000100:00080000:2.0:1769076441.759830:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076441.759837:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076441.759873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612607616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076441.759901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076441.759904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076441.759906:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076441.759909:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3429 00000100:00080000:2.0:1769076441.759913:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076441.759916:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076441.759924:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076441.759934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612607744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076441.759943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076441.759947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076441.759948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076441.759951:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3429 00000100:00080000:2.0:1769076441.759953:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076441.759959:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076441.759960:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612607616:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769076441.759963:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076441.759977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612607872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076441.759986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076441.759989:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076441.759991:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076441.759993:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3429 00000100:00080000:2.0:1769076441.759996:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076441.759999:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076441.760003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076441.760011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612608000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076441.760028:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076441.760064:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff8b1005d5b100 x1855007612607616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.760083:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612607616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.760092:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.760096:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076441.760099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612607616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076441.760105:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607744:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076441.760122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076441.760136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff8b1005d5aa00 x1855007612607744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.760152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612607744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.760163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.760167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076441.760172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076441.760200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076441.760220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076441.760230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff8b1005d58380 x1855007612607872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.760240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612607872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.760252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.760254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076441.760257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612607872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076441.760261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076441.760274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076441.760281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff8b1005d5ad80 x1855007612608000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.760290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612608000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.760296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.760298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076441.760301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076442.655812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076442.655907:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076443.679745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076443.679845:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076444.703807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076444.703920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076445.727815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076445.727928:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076446.751932:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076446.751983:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076446.879774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076446.879786:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076446.879791:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076446.879802:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3434 00000100:00080000:2.0:1769076446.879815:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076446.879823:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076446.879888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612608128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076446.879928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076446.879934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076446.879936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076446.879942:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3434 00000100:00080000:2.0:1769076446.879946:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076446.879951:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076446.879962:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076446.879989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612608256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076446.880003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076446.880010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076446.880014:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076446.880018:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3434 00000100:00080000:2.0:1769076446.880025:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076446.880029:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076446.880035:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076446.880049:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612608384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076446.880060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076446.880064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076446.880067:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076446.880072:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3434 00000100:00080000:2.0:1769076446.880077:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076446.880085:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076446.880099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612608128:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769076446.880100:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076446.880129:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612608512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076446.880237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076446.880298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076446] req@ffff8b1005d59500 x1855007612608128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076446.880328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612608128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076446.880345:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076446.880353:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076446.880359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612608128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076446.880371:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076446.880399:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076446.880414:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076446] req@ffff8b1005d59880 x1855007612608256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076446.880430:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612608256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076446.880441:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076446.880446:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076446.880450:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076446.880458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076446.880480:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076446.880492:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076446] req@ffff8b1005d5b480 x1855007612608384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076446.880507:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612608384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076446.880518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076446.880521:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076446.880525:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076446.880565:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076446.880588:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076446.880601:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076446] req@ffff8b1005d58e00 x1855007612608512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076446.880616:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612608512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076446.880627:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076446.880630:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076446.880634:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076447.775827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076447.775901:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076448.799809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076448.799909:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076449.823760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076449.823869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076450.847733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076450.847836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076451.871770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076451.871856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076451.999736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.999746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076451.999750:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.999760:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3440 00000100:00080000:3.0:1769076451.999770:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.999778:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076451.999813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612608640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.999840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.999843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076451.999845:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.999849:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3440 00000100:00080000:3.0:1769076451.999852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.999855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076451.999864:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076451.999877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612608768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.999886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.999890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076451.999891:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.999895:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3440 00000100:00080000:3.0:1769076451.999898:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076451.999903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612608640:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076451.999906:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076451.999916:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076451.999927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612608896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.999937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.999941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076451.999945:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.999950:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3440 00000100:00080000:3.0:1769076451.999955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.999961:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076451.999967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076451.999981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612609024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076451.999995:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076452.000045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076452] req@ffff8b10037dc000 x1855007612608640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076452.000068:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612608640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076452.000079:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076452.000083:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076452.000086:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612608640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076452.000093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076452.000117:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076452.000128:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076452] req@ffff8b10037dea00 x1855007612608768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076452.000139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612608768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076452.000152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076452.000155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076452.000158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076452.000180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076452.000195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076452.000203:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076452] req@ffff8b10037dd880 x1855007612608896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076452.000212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612608896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076452.000219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076452.000222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076452.000225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612608896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076452.000230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076452.000243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076452.000250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076452] req@ffff8b10037df100 x1855007612609024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076452.000260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612609024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076452.000267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076452.000269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076452.000272:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076452.895751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076452.895869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076453.919778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076453.919830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076454.943778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076454.943888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076455.967707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076455.967767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076456.991769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076456.991863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076457.119840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.119853:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076457.119859:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.119872:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3445 00000100:00080000:0.0:1769076457.119886:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.119895:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076457.119951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612609152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.120025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.120033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.120037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.120045:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3445 00000100:00080000:0.0:1769076457.120053:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.120060:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076457.120076:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.120132:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612609280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.120150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.120162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769076457.120164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612609152:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076457.120167:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.120183:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3445 00000100:00080000:0.0:1769076457.120190:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.120195:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076457.120203:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.120222:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612609408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.120245:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.120249:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076457.120252:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.120258:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3445 00000100:00080000:0.0:1769076457.120313:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076457.120325:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076457.120327:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076457.120343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.120381:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612609536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076457.120404:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff8b1006024e00 x1855007612609152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076457.120438:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612609152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076457.120455:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076457.120463:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076457.120469:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612609152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076457.120568:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609280:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076457.120591:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076457.120601:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff8b1006024380 x1855007612609280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076457.120610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612609280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076457.120617:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076457.120619:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076457.120622:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076457.120627:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076457.120640:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076457.120647:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff8b1006027480 x1855007612609408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076457.120698:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612609408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076457.120704:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076457.120706:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076457.120709:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076457.120713:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076457.120729:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076457.120736:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff8b1006024700 x1855007612609536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076457.120744:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612609536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076457.120750:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076457.120751:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076457.120753:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076458.015818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076458.015960:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076459.039769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076459.039862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076460.063754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076460.063856:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076461.087801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076461.087850:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076462.111820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076462.111957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076462.239773:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.239791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076462.239799:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.239816:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3450 00000100:00080000:0.0:1769076462.239832:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.239842:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076462.239891:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612609664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.239939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.239945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076462.239948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.239954:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3450 00000100:00080000:0.0:1769076462.239960:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.239965:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076462.239976:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076462.239999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612609792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.240025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.240031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076462.240034:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.240042:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3450 00000100:00100000:3.0:1769076462.240052:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612609664:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076462.240053:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.240066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076462.240074:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076462.240092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612609920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.240108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.240113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076462.240117:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.240124:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3450 00000100:00080000:0.0:1769076462.240131:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.240137:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076462.240143:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076462.240155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076462.240166:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612610048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076462.240532:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff8b1006024700 x1855007612609664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076462.240566:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612609664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076462.240582:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076462.240588:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076462.240593:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612609664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076462.240605:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076462.240846:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076462.240869:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff8b1006027480 x1855007612609792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076462.240887:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612609792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076462.240898:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076462.240903:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076462.240908:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076462.240946:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076462.240991:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076462.241018:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff8b1006024380 x1855007612609920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076462.241046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612609920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076462.241062:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076462.241066:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076462.241071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612609920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076462.241080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076462.241110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076462.241125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff8b1006024e00 x1855007612610048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076462.241141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612610048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076462.241151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076462.241156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076462.241160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076463.135802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076463.135882:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076464.159758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076464.159936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076465.183796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076465.183903:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076466.207783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076466.207876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076467.231770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076467.231920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076467.359783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076467.359800:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076467.359807:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076467.359823:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3455 00000100:00080000:2.0:1769076467.359840:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076467.359852:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076467.359902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612610176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076467.359943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076467.359949:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076467.359951:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076467.359957:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3455 00000100:00080000:2.0:1769076467.359962:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076467.359969:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076467.359980:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076467.359997:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612610304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076467.360018:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612610176:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076467.360027:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076467.360038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076467.360041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076467.360048:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3455 00000100:00080000:2.0:1769076467.360052:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076467.360057:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076467.360063:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076467.360078:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612610432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076467.360089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076467.360094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076467.360097:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076467.360101:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3455 00000100:00080000:2.0:1769076467.360136:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076467.360142:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076467.360148:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076467.360163:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076467.360166:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612610560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076467.360241:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff8b1005d5a300 x1855007612610176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076467.360280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612610176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076467.360294:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076467.360301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076467.360307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612610176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076467.360384:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610304:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076467.360423:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076467.360442:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff8b1005d5bb80 x1855007612610304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076467.360460:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612610304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076467.360471:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076467.360476:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076467.360480:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076467.360488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076467.360512:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076467.360525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff8b1005d58a80 x1855007612610432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076467.360540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612610432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076467.360551:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076467.360554:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076467.360558:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076467.360565:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076467.360585:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076467.360597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff8b1005d5b800 x1855007612610560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076467.360622:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612610560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076467.360632:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076467.360635:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076467.360680:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076468.255867:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076468.255917:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076469.279779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076469.279854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076470.303763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076470.303898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076471.327745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076471.327887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076472.351785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076472.351987:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076472.479768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076472.479784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076472.479791:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076472.479807:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3460 00000100:00080000:2.0:1769076472.479827:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076472.479838:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076472.479901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612610688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076472.479949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076472.479955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076472.479966:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076472.479972:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3460 00000100:00080000:2.0:1769076472.479977:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076472.479983:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076472.479999:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076472.480017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612610816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076472.480034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612610688:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076472.480037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076472.480049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076472.480054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076472.480063:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3460 00000100:00080000:2.0:1769076472.480067:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076472.480071:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076472.480077:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076472.480099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612610944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076472.480112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076472.480117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076472.480120:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076472.480124:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3460 00000100:00080000:2.0:1769076472.480128:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076472.480133:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076472.480139:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076472.480151:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612611072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076472.480191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076472.480279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff8b1005d59180 x1855007612610688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076472.480309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612610688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076472.480325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076472.480332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076472.480338:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612610688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076472.480376:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610816:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076472.480411:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076472.480429:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff8b1005d58000 x1855007612610816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076472.480479:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612610816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076472.480491:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076472.480496:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076472.480500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076472.480509:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076472.480537:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076472.480551:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff8b1005d5b800 x1855007612610944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076472.480567:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612610944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076472.480578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076472.480581:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076472.480585:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612610944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076472.480592:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076472.480613:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076472.480627:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff8b1005d58a80 x1855007612611072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076472.480704:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612611072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076472.480736:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076472.480743:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076472.480750:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076473.375715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076473.375783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076474.399797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076474.399860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076475.423833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076475.423917:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076476.447894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076476.447908:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076477.471780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076477.471904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076477.599861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.599873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076477.599879:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.599891:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3465 00000100:00080000:1.0:1769076477.599905:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.599914:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076477.599964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612611200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.600017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.600022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076477.600025:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.600030:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3465 00000100:00080000:1.0:1769076477.600034:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.600039:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076477.600050:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076477.600067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612611328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.600081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.600086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076477.600089:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.600094:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3465 00000100:00080000:1.0:1769076477.600098:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.600102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076477.600112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612611200:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076477.600113:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076477.600135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612611456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.600151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.600184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076477.600187:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.600193:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3465 00000100:00080000:1.0:1769076477.600199:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.600204:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076477.600211:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076477.600229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612611584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076477.600247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076477.600303:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff8b1006136a00 x1855007612611200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076477.600329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612611200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076477.600343:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076477.600349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076477.600355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612611200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076477.600365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076477.600392:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076477.600407:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff8b1006135180 x1855007612611328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076477.600424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612611328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076477.600443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076477.600447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076477.600467:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076477.600496:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076477.600522:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076477.600537:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff8b1006137800 x1855007612611456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076477.600552:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612611456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076477.600563:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076477.600567:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076477.600571:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076477.600578:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076477.600597:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076477.600609:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff8b1006136300 x1855007612611584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076477.600624:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612611584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076477.600634:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076477.600692:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076477.600699:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076478.495804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076478.495898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076479.519777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076479.519908:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076480.543770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076480.544020:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076481.567795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076481.567848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076482.591818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076482.591951:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076482.719907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076482.719923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076482.719930:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076482.719947:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3470 00000100:00080000:2.0:1769076482.719966:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076482.719978:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076482.720043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612611712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076482.720148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076482.720157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076482.720163:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076482.720172:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3470 00000100:00080000:2.0:1769076482.720179:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076482.720188:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076482.720209:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076482.720245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612611840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076482.720252:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612611712:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076482.720298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076482.720304:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076482.720307:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076482.720317:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3470 00000100:00080000:2.0:1769076482.720323:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076482.720330:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076482.720340:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076482.720366:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612611968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076482.720382:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076482.720392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076482.720398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076482.720400:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076482.720408:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3470 00000100:00080000:2.0:1769076482.720414:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076482.720422:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076482.720431:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076482.720450:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612612096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076482.720479:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff8b1005d5bb80 x1855007612611712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076482.720524:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612611712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076482.720547:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076482.720556:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076482.720565:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612611712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076482.720605:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611840:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076482.720730:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076482.720762:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff8b1005d5a300 x1855007612611840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076482.720791:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612611840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076482.720803:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076482.720809:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076482.720814:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076482.720824:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076482.720858:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076482.720874:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff8b1005d58e00 x1855007612611968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076482.720889:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612611968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076482.720900:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076482.720904:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076482.720908:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612611968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076482.720915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076482.720935:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076482.720948:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff8b1005d5b480 x1855007612612096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076482.720962:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612612096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076482.720972:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076482.720976:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076482.720980:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076483.615803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076483.615922:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076484.639828:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076484.639948:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076485.663769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076485.663911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076486.687769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076486.687932:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076487.711761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076487.711890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076487.839787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076487.839801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076487.839807:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076487.839820:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3475 00000100:00080000:1.0:1769076487.839834:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076487.839843:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076487.839889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612612224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076487.839927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076487.839932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076487.839935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076487.839941:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3475 00000100:00080000:1.0:1769076487.839945:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076487.839950:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076487.839960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076487.839975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612612352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076487.840012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076487.840017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076487.840021:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076487.840027:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3475 00000100:00080000:1.0:1769076487.840034:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076487.840041:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076487.840049:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076487.840065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612612480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076487.840081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612612224:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076487.840086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076487.840101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076487.840106:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076487.840112:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3475 00000100:00080000:1.0:1769076487.840116:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076487.840121:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076487.840127:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076487.840143:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612612608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076487.840206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076487.840256:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff8b1006135c00 x1855007612612224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076487.840302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612612224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076487.840318:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076487.840323:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076487.840329:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612612224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076487.840339:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076487.840378:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076487.840404:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff8b1006137b80 x1855007612612352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076487.840431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612612352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076487.840450:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076487.840457:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076487.840464:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076487.840504:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076487.840542:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076487.840567:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff8b1006135880 x1855007612612480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076487.840594:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612612480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076487.840612:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076487.840619:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076487.840626:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076487.840682:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076487.840718:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076487.840735:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff8b1006135500 x1855007612612608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076487.840751:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612612608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076487.840762:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076487.840766:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076487.840770:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076488.735756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076488.735904:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076489.759782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076489.759935:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076490.783782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076490.783881:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076491.807738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076491.807796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076492.831718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076492.831804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076492.959762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.959779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076492.959787:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.959801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3481 00000100:00080000:1.0:1769076492.959817:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.959827:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076492.959889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612612736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.959935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.959949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076492.959951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.959955:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3481 00000100:00080000:1.0:1769076492.959959:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.959969:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076492.959982:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076492.959997:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612612864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076492.960000:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612612736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076492.960010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.960014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076492.960018:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.960022:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3481 00000100:00080000:1.0:1769076492.960025:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.960028:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076492.960031:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076492.960043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612612992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.960050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.960053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076492.960054:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.960057:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3481 00000100:00080000:1.0:1769076492.960060:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.960062:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076492.960066:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076492.960074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612613120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076492.960098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076492.960143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff8b1006137100 x1855007612612736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.960159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612612736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.960169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.960173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076492.960177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612612736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076492.960209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076492.960227:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076492.960237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff8b1006134700 x1855007612612864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.960246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612612864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.960252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.960254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076492.960257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076492.960261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076492.960274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076492.960280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff8b1006136d80 x1855007612612992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.960293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612612992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.960299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.960301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076492.960303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612612992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076492.960307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076492.960318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076492.960325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff8b1006134e00 x1855007612613120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.960333:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612613120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.960338:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.960340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076492.960342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076493.855773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076493.855978:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076494.879769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076494.879951:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076495.903772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076495.903923:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076496.927774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076496.927915:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076497.951710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076497.951799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076498.079731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076498.079743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076498.079748:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076498.079758:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3486 00000100:00080000:1.0:1769076498.079770:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076498.079778:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076498.079825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612613248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076498.079847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076498.079850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076498.079852:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076498.079856:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3486 00000100:00080000:1.0:1769076498.079860:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076498.079863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076498.079871:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076498.079873:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612613248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076498.079887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612613376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076498.079898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076498.079903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076498.079904:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076498.079907:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3486 00000100:00080000:1.0:1769076498.079911:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076498.079915:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076498.079919:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076498.079929:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612613504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076498.079937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076498.079939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076498.079941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076498.079945:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3486 00000100:00080000:1.0:1769076498.079948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076498.079951:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076498.079956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076498.079966:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612613632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076498.079972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076498.080000:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076498.080018:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076498.975730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076498.975841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076499.999721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076499.999813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076501.023721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076501.023812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076501.151907:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:0.0:1769076501.151945:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076501.151960:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769076501.151990:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769076501.151996:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769076501.152001:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769076501.152069:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076501] req@ffff8b1006134e00 x1855007612613632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076501.152098:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612613632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076501.152110:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076501.152116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076501.152120:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076501.152129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076501] req@ffff8b1006134a80 x1855007612613376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076501.152142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612613376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076501.152151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076501.152154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076501.152158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076501.152167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076501] req@ffff8b1006137480 x1855007612613504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076501.152180:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612613504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076501.152188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076501.152191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076501.152194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076501.152201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076501] req@ffff8b1006136680 x1855007612613248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076501.152213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612613248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076501.152220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076501.152224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076501.152227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612613248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076502.047808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076502.047880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076503.071740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076503.071797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076503.199834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076503.199848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076503.199854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076503.199866:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3491 00000100:00080000:2.0:1769076503.199879:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076503.199888:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076503.199938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612613760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076503.199995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076503.200001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076503.200004:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076503.200009:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3491 00000100:00080000:2.0:1769076503.200013:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076503.200019:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076503.200030:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076503.200044:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612613888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076503.200056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076503.200061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076503.200063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076503.200083:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3491 00000100:00080000:2.0:1769076503.200085:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076503.200088:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076503.200091:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076503.200097:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612614016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076503.200103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076503.200106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076503.200107:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076503.200110:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3491 00000100:00080000:2.0:1769076503.200113:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076503.200116:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076503.200119:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076503.200125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612614144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076503.200137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612613760:192.168.204.102@tcp:250:kworker.0 00000800:00000100:3.0:1769076503.200214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076503.200265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff8b1005d59880 x1855007612613760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076503.200287:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612613760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076503.200299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076503.200304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076503.200309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612613760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076503.200340:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076503.200388:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076503.200415:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff8b1005d59500 x1855007612613888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076503.200440:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612613888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076503.200458:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076503.200465:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076503.200473:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612613888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076503.200720:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076503.200805:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076503.200832:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff8b1005d5ad80 x1855007612614016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076503.200860:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612614016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076503.200877:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076503.200885:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076503.200891:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076503.200941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076503.200999:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076503.201026:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff8b1005d58380 x1855007612614144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076503.201052:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612614144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076503.201068:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076503.201074:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076503.201080:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076504.095734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076504.095800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076505.119773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076505.119855:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076506.143741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076506.143800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076507.167736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076507.167788:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076508.191738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076508.191802:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076508.319718:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076508.319725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076508.319729:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076508.319736:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3496 00000100:00080000:2.0:1769076508.319746:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076508.319751:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076508.319784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612614272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076508.319807:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076508.319810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076508.319811:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076508.319814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3496 00000100:00080000:2.0:1769076508.319816:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076508.319819:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076508.319825:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076508.319834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612614400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076508.319841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076508.319845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076508.319846:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076508.319852:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3496 00000100:00100000:3.0:1769076508.319854:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612614272:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076508.319855:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076508.319861:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076508.319865:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076508.319874:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612614528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076508.319880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076508.319883:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076508.319884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076508.319887:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3496 00000100:00080000:2.0:1769076508.319890:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076508.319892:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076508.319896:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076508.319902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612614656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076508.319917:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076508.319949:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff8b1005d5aa00 x1855007612614272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076508.319968:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612614272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076508.319976:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076508.319980:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076508.319983:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612614272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076508.319988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614400:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076508.320002:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076508.320010:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff8b1005d5b100 x1855007612614400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076508.320019:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612614400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076508.320024:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076508.320026:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076508.320028:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076508.320047:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076508.320059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076508.320065:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff8b1005d58380 x1855007612614528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076508.320073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612614528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076508.320083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076508.320085:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076508.320088:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076508.320091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076508.320101:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076508.320107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff8b1005d5ad80 x1855007612614656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076508.320115:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612614656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076508.320120:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076508.320122:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076508.320124:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076509.215739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076509.215800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076510.239773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076510.239835:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076511.263790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076511.263832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076512.287715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076512.287778:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076513.311736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076513.311815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076513.439732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076513.439742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076513.439745:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076513.439753:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3501 00000100:00080000:1.0:1769076513.439762:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076513.439768:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076513.439800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612614784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076513.439823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076513.439825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076513.439827:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076513.439830:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3501 00000100:00080000:1.0:1769076513.439832:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076513.439834:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076513.439841:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076513.439850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612614912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076513.439857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076513.439860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076513.439861:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076513.439864:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3501 00000100:00080000:1.0:1769076513.439866:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076513.439868:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076513.439871:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076513.439878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612615040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076513.439885:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612614784:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076513.439887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076513.439894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076513.439898:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076513.439901:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3501 00000100:00080000:1.0:1769076513.439903:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076513.439906:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076513.439909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076513.439918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612615168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076513.439964:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076513.440001:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076513] req@ffff8b1006136d80 x1855007612614784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076513.440016:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612614784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076513.440024:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076513.440036:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076513.440040:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612614784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076513.440046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076513.440064:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076513.440074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076513] req@ffff8b1006134700 x1855007612614912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076513.440083:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612614912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076513.440089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076513.440091:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076513.440094:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612614912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076513.440118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076513.440132:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076513.440140:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076513] req@ffff8b1006137100 x1855007612615040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076513.440148:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612615040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076513.440154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076513.440156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076513.440159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076513.440162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076513.440173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076513.440187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076513] req@ffff8b1006135500 x1855007612615168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076513.440195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612615168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076513.440200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076513.440202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076513.440204:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076514.335736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076514.335799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076515.359724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076515.359825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076516.383765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076516.383860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076517.407710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076517.407778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076518.431754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076518.431793:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076518.559764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.559773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076518.559776:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.559784:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3506 00000100:00080000:2.0:1769076518.559794:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.559799:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076518.559837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612615296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.559857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.559860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076518.559862:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.559865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3506 00000100:00080000:2.0:1769076518.559867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.559870:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076518.559878:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076518.559887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612615424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.559895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.559897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076518.559898:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.559900:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3506 00000100:00080000:2.0:1769076518.559903:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.559905:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076518.559912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076518.559922:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612615296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076518.559930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612615552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.559946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.559950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076518.559953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.559958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3506 00000100:00080000:2.0:1769076518.559962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.559967:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076518.559974:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076518.559989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612615680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076518.560010:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076518.560054:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff8b1005d59500 x1855007612615296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.560071:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612615296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.560080:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.560084:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076518.560087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612615296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076518.560094:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076518.560111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076518.560120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff8b1005d59880 x1855007612615424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.560129:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612615424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.560135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.560137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076518.560139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076518.560172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076518.560187:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076518.560194:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff8b1005d5b480 x1855007612615552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.560203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612615552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.560209:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.560212:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076518.560214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076518.560218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076518.560229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076518.560235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff8b1005d58e00 x1855007612615680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.560243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612615680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.560253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.560256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076518.560258:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076519.455798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076519.455924:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076520.479788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076520.479946:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076521.503720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076521.503954:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076522.527785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076522.527919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076523.551752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076523.551887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076523.679795:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076523.679812:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076523.679819:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076523.679835:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3511 00000100:00080000:3.0:1769076523.679851:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076523.679875:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076523.679931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612615808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076523.679984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076523.679992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076523.679996:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076523.680005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3511 00000100:00080000:3.0:1769076523.680014:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076523.680023:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076523.680041:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076523.680043:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612615808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076523.680079:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612615936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076523.680106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076523.680113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076523.680118:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076523.680127:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3511 00000100:00080000:3.0:1769076523.680137:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076523.680138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076523.680154:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076523.680169:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076523.680197:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff8b10037ddc00 x1855007612615808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076523.680202:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612616064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076523.680253:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076523.680259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612615808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076523.680264:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076523.680274:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076523.680289:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.680294:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3511 00000100:00080000:2.0:1769076523.680304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00080000:3.0:1769076523.680311:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076523.680314:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612615808:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076523.680330:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076523.680343:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076523.680355:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076523.680368:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612616192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076523.680429:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076523.680488:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff8b10037dce00 x1855007612615936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076523.680518:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612615936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076523.680535:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076523.680543:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076523.680550:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612615936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076523.680563:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076523.680611:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076523.680673:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff8b10037df480 x1855007612616064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076523.680703:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612616064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076523.680720:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076523.680727:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076523.680734:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076523.680765:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076523.680815:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076523.680832:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff8b10037df800 x1855007612616192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076523.680849:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612616192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076523.680859:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076523.680863:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076523.680868:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076524.575746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076524.575954:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076525.599849:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076525.599933:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076526.623802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076526.623962:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076527.647810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076527.647917:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076528.671761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076528.671815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076528.799755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076528.799764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076528.799768:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076528.799776:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3516 00000100:00080000:0.0:1769076528.799786:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076528.799791:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076528.799823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612616320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076528.799846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076528.799849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076528.799850:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076528.799853:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3516 00000100:00080000:0.0:1769076528.799856:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076528.799858:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076528.799865:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076528.799873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612616448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076528.799880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076528.799882:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076528.799884:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076528.799886:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3516 00000100:00080000:0.0:1769076528.799888:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076528.799890:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076528.799892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076528.799898:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612616576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076528.799905:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076528.799908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076528.799909:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076528.799911:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3516 00000100:00080000:0.0:1769076528.799913:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076528.799916:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076528.799918:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076528.799927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612616704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076528.799929:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612616320:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769076528.800004:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076528.800045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff8b1006024a80 x1855007612616320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076528.800060:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612616320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076528.800067:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076528.800071:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076528.800073:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612616320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076528.800079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076528.800093:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076528.800101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff8b1006025f80 x1855007612616448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076528.800109:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612616448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076528.800123:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076528.800125:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076528.800127:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076528.800131:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076528.800143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076528.800149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff8b1006027b80 x1855007612616576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076528.800156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612616576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076528.800162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076528.800163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076528.800165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076528.800196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076528.800227:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076528.800235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff8b1006026d80 x1855007612616704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076528.800248:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612616704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076528.800254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076528.800256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076528.800258:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076529.695762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076529.695895:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076530.719817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076530.719943:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076531.743787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076531.743919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076532.767851:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076532.767882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076533.791700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076533.791852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076533.919781:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.919788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076533.919792:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.919802:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3522 00000100:00080000:3.0:1769076533.919812:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.919819:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076533.919851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612616832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076533.919875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.919878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076533.919880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.919883:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3522 00000100:00080000:3.0:1769076533.919886:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.919890:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076533.919897:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076533.919911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612616960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076533.919916:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612616832:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076533.919924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.919928:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076533.919929:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.919935:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3522 00000100:00080000:3.0:1769076533.919942:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.919945:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076533.919949:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076533.919961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612617088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076533.919973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.919976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076533.919979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076533.919984:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076533.919987:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3522 00000100:00080000:3.0:1769076533.919993:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.919997:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076533.920002:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076533.920010:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612617216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076533.920022:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff8b10037dc700 x1855007612616832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.920040:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612616832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.920049:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.920054:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076533.920058:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612616832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076533.920092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076533.920111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076533.920122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff8b10037df800 x1855007612616960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.920132:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612616960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.920138:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.920140:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076533.920142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612616960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076533.920148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076533.920159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076533.920166:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff8b10037df480 x1855007612617088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.920176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612617088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.920186:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.920189:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076533.920192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076533.920200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076533.920238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076533.920252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff8b10037dce00 x1855007612617216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.920279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612617216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.920287:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.920290:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076533.920293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076534.815710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076534.815789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076535.839726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076535.839788:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076536.863741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076536.863829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076537.887751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076537.887883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076538.911781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076538.911847:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076539.039754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.039764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076539.039768:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.039776:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3527 00000100:00080000:1.0:1769076539.039786:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.039791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076539.039823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612617344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.039847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.039850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.039852:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.039854:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3527 00000100:00080000:1.0:1769076539.039857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.039859:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076539.039865:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.039873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612617472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076539.039880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.039885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076539.039887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.039890:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3527 00000100:00080000:1.0:1769076539.039892:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.039895:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076539.039898:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.039908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612617600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076539.039909:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612617344:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076539.039923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076539.039925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076539.039926:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076539.039929:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3527 00000100:00080000:1.0:1769076539.039931:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076539.039934:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076539.039937:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076539.039944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612617728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076539.039975:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076539.040005:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff8b1006135880 x1855007612617344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.040019:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612617344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.040027:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.040030:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076539.040033:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612617344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076539.040038:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617472:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076539.040051:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076539.040059:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff8b1006137b80 x1855007612617472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.040067:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612617472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.040072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.040074:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076539.040076:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076539.040093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.040105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076539.040111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff8b1006135c00 x1855007612617600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.040149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612617600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.040156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.040158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076539.040160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076539.040164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076539.040179:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076539.040186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076539/real 1769076539] req@ffff8b1006136300 x1855007612617728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076539.040194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612617728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076539.040199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076539.040200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076539.040202:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617728:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769076539.935874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076539.935881:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076540.959758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076540.959909:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076541.983756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076541.983886:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076543.007727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076543.007787:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076544.031736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076544.031787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076544.159825:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076544.159836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076544.159842:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076544.159854:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3532 00000100:00080000:3.0:1769076544.159869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076544.159877:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076544.159925:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612617856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076544.159964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076544.159970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076544.159973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076544.159978:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3532 00000100:00080000:3.0:1769076544.159983:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076544.159987:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076544.159998:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076544.160013:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612617984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076544.160026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076544.160030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076544.160033:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076544.160104:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3532 00000100:00100000:0.0:1769076544.160113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612617856:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076544.160115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076544.160127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076544.160135:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076544.160155:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612618112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076544.160170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076544.160175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076544.160178:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076544.160182:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3532 00000100:00080000:3.0:1769076544.160187:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076544.160193:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076544.160200:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076544.160208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076544.160217:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612618240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076544.160264:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff8b10037ddc00 x1855007612617856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.160293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612617856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.160307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.160313:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076544.160318:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612617856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076544.160328:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076544.160353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076544.160367:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff8b10037df100 x1855007612617984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.160384:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612617984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.160394:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.160398:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076544.160402:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612617984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076544.160431:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076544.160452:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076544.160463:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff8b10037dd880 x1855007612618112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.160478:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612618112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.160496:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.160500:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076544.160504:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076544.160511:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076544.160530:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076544.160541:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff8b10037dea00 x1855007612618240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.160555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612618240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.160566:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.160569:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076544.160573:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076545.055764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076545.055885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076546.079733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076546.079807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076547.103723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076547.103777:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076548.127773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076548.127834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076549.151722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076549.151789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076549.279801:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076549.279810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076549.279814:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076549.279822:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3537 00000100:00080000:0.0:1769076549.279832:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076549.279838:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076549.279873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612618368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076549.279899:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076549.279902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076549.279903:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076549.279907:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3537 00000100:00080000:0.0:1769076549.279909:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076549.279911:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076549.279918:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076549.279928:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612618496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076549.279935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076549.279938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076549.279939:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076549.279941:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3537 00000100:00080000:0.0:1769076549.279944:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076549.279946:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076549.279970:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076549.279979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612618624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076549.279981:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612618368:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076549.279991:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076549.279994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076549.279995:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076549.279998:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3537 00000100:00080000:0.0:1769076549.280000:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076549.280003:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076549.280007:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076549.280016:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612618752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076549.280056:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076549.280089:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff8b1006026300 x1855007612618368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076549.280103:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612618368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076549.280110:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076549.280113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076549.280116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612618368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076549.280122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618496:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076549.280138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076549.280146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff8b1006024000 x1855007612618496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076549.280155:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612618496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076549.280160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076549.280162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076549.280164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076549.280182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076549.280193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076549.280199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff8b1006025880 x1855007612618624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076549.280207:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612618624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076549.280212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076549.280214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076549.280216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076549.280220:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076549.280230:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076549.280236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff8b1006026a00 x1855007612618752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076549.280243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612618752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076549.280249:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076549.280250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076549.280252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612618752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076550.175711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076550.175772:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076551.199733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076551.199794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076552.223771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076552.223852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076553.247728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076553.247825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076554.271755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076554.271844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076554.399784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076554.399796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076554.399801:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076554.399812:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3542 00000100:00080000:0.0:1769076554.399826:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076554.399834:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076554.399913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612618880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076554.399952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076554.399956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076554.399958:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076554.399963:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3542 00000100:00080000:0.0:1769076554.399968:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076554.399972:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076554.399983:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076554.399992:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612618880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076554.400004:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612619008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076554.400015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076554.400019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076554.400022:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076554.400027:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3542 00000100:00080000:0.0:1769076554.400030:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076554.400032:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076554.400036:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076554.400046:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612619136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076554.400053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076554.400056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076554.400058:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076554.400060:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3542 00000100:00080000:0.0:1769076554.400063:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076554.400065:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769076554.400072:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769076554.400074:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076554.400087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612619264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076554.400110:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff8b1006027800 x1855007612618880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076554.400128:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612618880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076554.400137:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076554.400141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076554.400144:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612618880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076554.400165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076554.400181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076554.400190:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff8b1006027100 x1855007612619008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076554.400198:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612619008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076554.400208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076554.400211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076554.400213:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619008:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076554.400217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076554.400229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076554.400235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff8b1006026680 x1855007612619136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076554.400243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612619136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076554.400249:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076554.400251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076554.400253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076554.400257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076554.400267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076554.400273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff8b1006025180 x1855007612619264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076554.400280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612619264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076554.400286:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076554.400288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076554.400291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076555.295724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076555.295814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076556.319741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076556.319806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076557.343764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076557.343832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076558.367712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076558.367789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076559.391720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076559.391818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076559.519764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.519774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076559.519805:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.519816:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3547 00000100:00080000:2.0:1769076559.519826:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.519832:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076559.519864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612619392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076559.519889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.519893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076559.519895:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.519899:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3547 00000100:00080000:2.0:1769076559.519903:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.519907:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076559.519917:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076559.519923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612619392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076559.519929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612619520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076559.519939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.519942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076559.519944:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.519947:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3547 00000100:00080000:2.0:1769076559.519951:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.519955:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076559.519959:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076559.519968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612619648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076559.519976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.519979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076559.519981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769076559.519982:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076559.519984:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3547 00000100:00080000:2.0:1769076559.519987:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.519990:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076559.519994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076559.520002:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612619776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076559.520015:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff8b1005d5a300 x1855007612619392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.520033:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612619392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.520044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.520048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076559.520051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612619392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076559.520070:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076559.520088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076559.520097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff8b1005d5bb80 x1855007612619520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.520106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612619520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.520111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.520114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076559.520116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076559.520119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076559.520131:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076559.520137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff8b1005d58a80 x1855007612619648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.520145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612619648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.520150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.520152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076559.520154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076559.520157:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076559.520168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076559.520173:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff8b1005d5b800 x1855007612619776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.520181:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612619776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.520186:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.520187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076559.520189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612619776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076560.415727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076560.415821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076561.439730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076561.439781:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076562.463769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076562.463832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076563.487744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076563.487779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076564.511732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076564.511786:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076564.639772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076564.639781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076564.639785:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076564.639792:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3552 00000100:00080000:0.0:1769076564.639803:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076564.639809:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076564.639844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612619904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076564.639867:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076564.639870:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076564.639871:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076564.639874:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3552 00000100:00080000:0.0:1769076564.639877:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076564.639880:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076564.639887:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076564.639898:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612620032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076564.639904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612619904:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076564.639909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076564.639912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076564.639914:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076564.639917:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3552 00000100:00080000:0.0:1769076564.639919:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076564.639922:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076564.639925:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076564.639934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612620160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076564.639940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076564.639943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076564.639944:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076564.639947:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3552 00000100:00080000:0.0:1769076564.639949:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076564.639952:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076564.639955:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076564.639964:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612620288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076564.639998:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076564.640020:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076564.640036:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076565.535717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076565.535801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076566.559743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076566.559811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076567.583723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076567.583773:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076567.711854:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076567.711891:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076567.711906:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076567.711939:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076567.711949:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076567.711958:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769076567.711985:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076567] req@ffff8b1006026680 x1855007612620032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076567.712012:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612620032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076567.712023:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076567.712028:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076567.712033:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076567.712041:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076567] req@ffff8b1006027100 x1855007612620160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076567.712055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612620160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076567.712064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076567.712068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076567.712071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076567.712077:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076567] req@ffff8b1006027800 x1855007612620288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076567.712088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612620288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076567.712096:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076567.712099:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076567.712102:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076567.712109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076567] req@ffff8b1006025180 x1855007612619904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076567.712119:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612619904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076567.712126:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076567.712129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076567.712132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612619904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076568.607757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076568.607829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076569.631777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076569.631886:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076569.759836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.759847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076569.759851:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.759862:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3557 00000100:00080000:2.0:1769076569.759875:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.759894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076569.759938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612620416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.759988:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.759994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076569.759997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.760003:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3557 00000100:00080000:2.0:1769076569.760019:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.760028:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076569.760042:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076569.760067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612620544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076569.760073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612620416:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076569.760105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.760110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076569.760115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.760122:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3557 00000100:00080000:2.0:1769076569.760128:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.760134:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076569.760143:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076569.760162:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612620672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.760178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076569.760183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076569.760185:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076569.760190:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.760198:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3557 00000100:00080000:2.0:1769076569.760206:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.760213:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076569.760221:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076569.760250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff8b1005d58000 x1855007612620416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076569.760254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612620800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076569.760314:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612620416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076569.760331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076569.760345:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076569.760361:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612620416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076569.760399:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076569.760452:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076569.760473:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff8b1005d59180 x1855007612620544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076569.760505:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612620544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076569.760518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076569.760523:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076569.760528:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076569.760538:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076569.760572:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076569.760593:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff8b1005d5b800 x1855007612620672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076569.760711:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612620672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076569.760727:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076569.760732:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076569.760738:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076569.760748:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076569.760782:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076569.760800:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff8b1005d58a80 x1855007612620800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076569.760819:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612620800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076569.760840:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076569.760845:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076569.760850:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612620800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076570.655755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076570.655887:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076571.679784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076571.679914:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076572.703784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076572.703887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076573.727746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076573.727932:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076574.751754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076574.751942:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076574.879835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076574.879852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076574.879860:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076574.879875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3562 00000100:00080000:0.0:1769076574.879893:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076574.879903:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076574.879993:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612620928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076574.880033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076574.880039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076574.880041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076574.880047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3562 00000100:00080000:0.0:1769076574.880052:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076574.880057:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076574.880081:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076574.880105:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612621056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076574.880107:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612620928:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076574.880143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076574.880148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076574.880151:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076574.880157:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3562 00000100:00080000:0.0:1769076574.880162:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076574.880166:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076574.880172:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076574.880189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612621184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076574.880199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076574.880204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076574.880206:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076574.880211:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3562 00000100:00080000:0.0:1769076574.880220:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076574.880225:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076574.880231:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076574.880240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076574.880251:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612621312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076574.880310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff8b1006026a00 x1855007612620928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076574.880337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612620928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076574.880363:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076574.880369:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076574.880374:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612620928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076574.880408:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621056:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076574.880435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076574.880450:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff8b1006025880 x1855007612621056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076574.880465:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612621056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076574.880476:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076574.880480:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076574.880492:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076574.880499:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076574.880519:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076574.880590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff8b1006024000 x1855007612621184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076574.880615:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612621184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076574.880682:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076574.880687:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076574.880692:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076574.880700:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076574.880732:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076574.880747:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff8b1006026300 x1855007612621312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076574.880762:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612621312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076574.880772:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076574.880776:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076574.880780:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076575.775733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076575.775845:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076576.799769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076576.799911:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076577.823717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076577.823811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076578.847727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076578.847779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076579.871737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076579.871872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076579.999831:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076579.999849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076579.999857:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076579.999872:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3568 00000100:00080000:0.0:1769076579.999900:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076579.999922:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076579.999993:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612621440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076580.000033:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076580.000039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076580.000041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076580.000047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3568 00000100:00080000:0.0:1769076580.000051:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076580.000057:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076580.000072:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076580.000090:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612621568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076580.000102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612621440:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076580.000116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076580.000125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076580.000130:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076580.000141:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3568 00000100:00080000:0.0:1769076580.000149:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076580.000157:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076580.000168:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076580.000191:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612621696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076580.000225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076580.000242:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076580.000267:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076580.000271:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076580.000279:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3568 00000100:00080000:0.0:1769076580.000287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076580.000294:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076580.000301:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076580.000306:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076580] req@ffff8b1006026d80 x1855007612621440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076580.000324:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612621824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076580.000360:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612621440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.000377:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076580.000387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076580.000391:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612621440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076580.000422:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076580.000497:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076580.000529:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076580] req@ffff8b1006027b80 x1855007612621568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076580.000545:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612621568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.000555:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076580.000559:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076580.000563:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076580.000572:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076580.000595:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076580.000606:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076580] req@ffff8b1006025f80 x1855007612621696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076580.000619:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612621696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.000647:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076580.000651:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076580.000655:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076580.000663:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076580.000751:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076580.000770:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076580/real 1769076580] req@ffff8b1006024a80 x1855007612621824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076580.000815:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612621824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076580.000840:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076580.000846:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076580.000850:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612621824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076580.895791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076580.895916:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076581.919706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076581.919769:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076582.943786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076582.943831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076583.967765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076583.967915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076584.991747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076584.991986:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076585.119856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076585.119868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076585.119873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076585.119884:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3573 00000100:00080000:0.0:1769076585.119898:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076585.119907:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076585.119957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612621952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076585.120008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076585.120015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076585.120019:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076585.120043:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3573 00000100:00080000:0.0:1769076585.120051:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076585.120058:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076585.120072:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076585.120092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612622080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076585.120105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076585.120126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076585.120128:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076585.120133:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3573 00000100:00080000:0.0:1769076585.120139:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076585.120144:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076585.120151:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076585.120166:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612622208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076585.120175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612621952:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076585.120188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076585.120194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076585.120196:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076585.120221:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3573 00000100:00080000:0.0:1769076585.120226:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076585.120230:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076585.120236:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076585.120254:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612622336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076585.120290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076585.120336:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff8b1006024e00 x1855007612621952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076585.120424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612621952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076585.120438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076585.120444:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076585.120449:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612621952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076585.120459:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076585.120489:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076585.120504:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff8b1006024380 x1855007612622080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076585.120547:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612622080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076585.120557:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076585.120561:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076585.120565:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076585.120593:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076585.120616:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076585.120628:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff8b1006027480 x1855007612622208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076585.120642:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612622208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076585.120651:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076585.120655:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076585.120659:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076585.120665:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076585.120722:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076585.120736:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff8b1006024700 x1855007612622336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076585.120750:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612622336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076585.120777:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076585.120780:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076585.120784:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076586.015753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076586.015905:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076587.039741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076587.039898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076588.063723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076588.063785:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076589.087734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076589.087798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076590.111788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076590.111993:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076590.239836:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076590.239852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076590.239858:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076590.239872:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3578 00000100:00080000:1.0:1769076590.239890:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076590.239903:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076590.239987:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612622464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076590.240023:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076590.240027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076590.240030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076590.240035:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3578 00000100:00080000:1.0:1769076590.240040:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076590.240045:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076590.240058:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076590.240078:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612622592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076590.240092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076590.240096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076590.240100:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076590.240104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612622464:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076590.240108:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3578 00000100:00080000:1.0:1769076590.240120:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076590.240124:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076590.240130:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076590.240146:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612622720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076590.240159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076590.240163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076590.240165:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076590.240169:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3578 00000100:00080000:1.0:1769076590.240173:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076590.240178:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076590.240183:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076590.240195:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612622848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076590.240219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076590.240292:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff8b1006137800 x1855007612622464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076590.240328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612622464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076590.240342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076590.240348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076590.240353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612622464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076590.240385:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622592:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076590.240412:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076590.240438:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff8b1006135180 x1855007612622592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076590.240453:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612622592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076590.240463:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076590.240468:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076590.240471:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076590.240478:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076590.240500:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076590.240511:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff8b1006136a00 x1855007612622720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076590.240524:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612622720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076590.240533:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076590.240536:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076590.240539:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076590.240545:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076590.240565:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076590.240576:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff8b1006136300 x1855007612622848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076590.240588:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612622848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076590.240597:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076590.240599:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076590.240603:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612622848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076591.135733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076591.135927:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076592.159700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076592.159796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076593.183686:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076593.183778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076594.207708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076594.207783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076595.231727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076595.231833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076595.359750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076595.359761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076595.359766:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076595.359776:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3583 00000100:00080000:2.0:1769076595.359787:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076595.359795:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076595.359828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612622976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076595.359855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076595.359859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076595.359861:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076595.359865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3583 00000100:00080000:2.0:1769076595.359869:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076595.359873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076595.359883:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076595.359897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612623104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076595.359909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076595.359912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076595.359913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076595.359916:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3583 00000100:00080000:2.0:1769076595.359919:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076595.359921:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076595.359927:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076595.359934:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612622976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076595.359938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612623232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076595.359947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076595.359949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076595.359950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076595.359953:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3583 00000100:00080000:2.0:1769076595.359956:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076595.359958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076595.359961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076595.359969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612623360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076595.360008:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076595.360039:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff8b1005d5bb80 x1855007612622976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076595.360053:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612622976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076595.360061:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076595.360064:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076595.360067:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612622976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076595.360072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076595.360087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076595.360095:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff8b1005d5a300 x1855007612623104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076595.360103:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612623104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076595.360109:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076595.360110:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076595.360113:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076595.360131:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076595.360143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076595.360149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff8b1005d58e00 x1855007612623232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076595.360156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612623232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076595.360161:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076595.360163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076595.360165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076595.360169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076595.360195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076595.360204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff8b1005d5b480 x1855007612623360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076595.360212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612623360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076595.360217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076595.360219:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076595.360221:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076596.255701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076596.255775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076597.279783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076597.279911:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076598.303735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076598.303945:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076599.327731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076599.327857:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076600.351773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076600.351998:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076600.479789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076600.479802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076600.479808:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076600.479821:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3588 00000100:00080000:3.0:1769076600.479835:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076600.479844:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076600.479887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612623488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076600.479928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076600.479932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076600.479934:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076600.479952:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3588 00000100:00080000:3.0:1769076600.479956:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076600.479960:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076600.479971:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076600.479990:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612623616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076600.480002:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612623488:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076600.480012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076600.480019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076600.480022:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076600.480029:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3588 00000100:00080000:3.0:1769076600.480044:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076600.480050:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076600.480058:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076600.480077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612623744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076600.480118:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076600.480128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076600.480134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076600.480137:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076600.480142:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3588 00000100:00080000:3.0:1769076600.480147:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076600.480151:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076600.480158:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076600.480170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff8b10037dc000 x1855007612623488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076600.480173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612623872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076600.480218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612623488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.480230:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.480235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076600.480238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612623488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076600.480260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076600.480284:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076600.480296:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff8b10037dd500 x1855007612623616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.480307:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612623616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.480315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.480318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076600.480321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076600.480326:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076600.480341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076600.480350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff8b10037ddf80 x1855007612623744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.480360:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612623744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.480367:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.480369:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076600.480372:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076600.480377:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076600.480390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076600.480398:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff8b10037de300 x1855007612623872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.480409:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612623872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.480416:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.480420:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076600.480424:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612623872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076601.375755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076601.375938:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076602.399836:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076602.400114:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076603.423777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076603.423903:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076604.447844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076604.447888:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076605.471762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076605.471883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076605.599718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.599730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076605.599735:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.599758:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3593 00000100:00080000:3.0:1769076605.599772:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.599782:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076605.599835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612624000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.599879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.599883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076605.599886:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.599892:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3593 00000100:00080000:3.0:1769076605.599911:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.599917:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076605.599928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076605.599947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612624128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.599958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.599962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076605.599964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.599970:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3593 00000100:00080000:3.0:1769076605.599975:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.599980:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076605.599985:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076605.600003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612624256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.600037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076605.600038:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612624000:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076605.600048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076605.600054:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.600061:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3593 00000100:00080000:3.0:1769076605.600067:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.600071:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076605.600078:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076605.600100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612624384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076605.600133:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076605.600217:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff8b10037dd180 x1855007612624000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.600253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612624000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.600265:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.600271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076605.600278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612624000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076605.600289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624128:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076605.600321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076605.600353:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff8b10037ded80 x1855007612624128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.600376:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612624128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.600391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.600406:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076605.600413:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076605.600464:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076605.600491:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076605.600504:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff8b10037de680 x1855007612624256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.600518:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612624256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.600534:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.600538:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076605.600542:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076605.600549:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076605.600567:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076605.600578:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff8b10037dc380 x1855007612624384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.600591:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612624384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.600600:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.600603:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076605.600606:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076606.495770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076606.495893:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076607.519741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076607.519942:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076608.543830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076608.543944:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076609.567777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076609.567880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076610.591795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076610.591896:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076610.719822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076610.719833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076610.719837:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076610.719846:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3598 00000100:00080000:0.0:1769076610.719857:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076610.719864:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076610.719901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612624512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076610.719959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076610.719964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076610.719965:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076610.719969:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3598 00000100:00080000:0.0:1769076610.719973:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076610.719977:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076610.719986:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076610.719999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612624640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076610.720010:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612624512:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076610.720012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076610.720020:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076610.720024:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076610.720028:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3598 00000100:00080000:0.0:1769076610.720031:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076610.720034:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076610.720038:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076610.720049:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612624768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076610.720057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076610.720060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076610.720062:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076610.720065:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3598 00000100:00080000:0.0:1769076610.720069:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076610.720072:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076610.720076:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076610.720084:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612624896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076610.720098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076610.720140:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff8b1006024700 x1855007612624512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076610.720166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612624512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076610.720180:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076610.720186:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076610.720192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612624512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076610.720217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076610.720254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076610.720266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff8b1006027480 x1855007612624640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076610.720277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612624640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076610.720284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076610.720287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076610.720290:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076610.720295:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076610.720311:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076610.720319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff8b1006024380 x1855007612624768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076610.720329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612624768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076610.720336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076610.720338:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076610.720340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076610.720345:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076610.720358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076610.720366:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff8b1006024e00 x1855007612624896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076610.720376:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612624896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076610.720382:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076610.720384:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076610.720387:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612624896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076611.615774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076611.615945:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076612.639732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076612.639804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076613.663801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076613.663963:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076614.687835:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076614.687911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076615.711776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076615.711913:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076615.839920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076615.839931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076615.839937:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076615.839948:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3603 00000100:00080000:1.0:1769076615.839962:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076615.839971:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076615.840015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612625024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076615.840065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076615.840070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076615.840073:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076615.840078:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3603 00000100:00080000:1.0:1769076615.840082:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076615.840086:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076615.840099:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076615.840113:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612625152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076615.840127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076615.840132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076615.840136:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076615.840140:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3603 00000100:00080000:1.0:1769076615.840144:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076615.840148:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076615.840154:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076615.840166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612625280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076615.840182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612625024:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076615.840184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076615.840197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076615.840204:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076615.840210:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3603 00000100:00080000:1.0:1769076615.840213:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076615.840217:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076615.840223:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076615.840240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612625408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076615.840301:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076615.840348:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff8b1006135c00 x1855007612625024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076615.840373:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612625024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076615.840396:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076615.840402:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076615.840408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612625024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076615.840418:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625152:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076615.840444:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076615.840459:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff8b1006137b80 x1855007612625152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076615.840484:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612625152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076615.840494:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076615.840498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076615.840502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076615.840532:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076615.840554:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076615.840567:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff8b1006135880 x1855007612625280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076615.840582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612625280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076615.840592:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076615.840595:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076615.840599:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076615.840606:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076615.840625:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076615.840636:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff8b1006135500 x1855007612625408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076615.840698:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612625408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076615.840717:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076615.840721:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076615.840725:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076616.735790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076616.735947:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076617.759847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076617.760016:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076618.783774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076618.783930:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076619.807726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076619.807794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076620.831804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076620.831925:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076620.959868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076620.959883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076620.959888:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076620.959900:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3609 00000100:00080000:3.0:1769076620.959915:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076620.959924:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076620.959974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612625536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076620.960019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076620.960027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076620.960031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076620.960039:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3609 00000100:00080000:3.0:1769076620.960047:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076620.960055:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076620.960089:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076620.960113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612625536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076620.960125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612625664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076620.960159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076620.960164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076620.960167:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076620.960173:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3609 00000100:00080000:3.0:1769076620.960181:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076620.960187:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076620.960196:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076620.960215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612625792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076620.960231:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076620.960235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076620.960243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076620.960245:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076620.960252:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3609 00000100:00080000:3.0:1769076620.960259:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076620.960264:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076620.960270:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076620.960286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612625920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076620.960313:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff8b10037dca80 x1855007612625536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076620.960354:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612625536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076620.960375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076620.960384:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076620.960392:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612625536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076620.960433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625664:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076620.960472:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076620.960491:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff8b10037dc380 x1855007612625664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076620.960508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612625664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076620.960519:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076620.960524:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076620.960529:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076620.960537:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076620.960560:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076620.960572:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff8b10037de680 x1855007612625792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076620.960587:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612625792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076620.960597:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076620.960600:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076620.960604:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076620.960611:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076620.960630:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076620.960690:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff8b10037ded80 x1855007612625920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076620.960707:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612625920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076620.960717:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076620.960721:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076620.960726:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612625920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076621.855768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076621.855873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076622.879712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076622.879816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076623.903840:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076623.903999:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076624.927858:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076624.927957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076625.951821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076625.951903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076626.079763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.079775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076626.079781:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.079793:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3614 00000100:00080000:3.0:1769076626.079808:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.079817:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076626.079863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612626048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076626.079903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.079908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076626.079911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.079916:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3614 00000100:00080000:3.0:1769076626.079920:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.079924:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076626.079935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076626.079950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612626176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076626.079964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.079969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076626.079973:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.079979:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3614 00000100:00080000:3.0:1769076626.079983:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.079989:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076626.079995:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076626.080007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612626304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076626.080022:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612626048:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076626.080025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.080040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076626.080043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.080048:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3614 00000100:00080000:3.0:1769076626.080052:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.080056:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076626.080063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076626.080078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612626432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076626.080140:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076626.080196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff8b10037dd180 x1855007612626048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076626.080223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612626048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076626.080238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076626.080243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076626.080249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612626048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076626.080259:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626176:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076626.080292:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076626.080307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff8b10037de300 x1855007612626176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076626.080324:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612626176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076626.080334:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076626.080337:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076626.080342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626176:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076626.080373:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076626.080396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076626.080408:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff8b10037ddf80 x1855007612626304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076626.080425:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612626304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076626.080435:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076626.080438:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076626.080442:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076626.080449:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076626.080468:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076626.080480:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff8b10037dd500 x1855007612626432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076626.080501:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612626432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076626.080511:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076626.080514:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076626.080518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076626.975805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076626.975912:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076627.999831:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076627.999893:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076629.023754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076629.023829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076630.047729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076630.047835:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076631.071699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076631.071807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076631.199792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076631.199801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076631.199806:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076631.199815:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3619 00000100:00080000:1.0:1769076631.199825:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076631.199831:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076631.199867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612626560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076631.199908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076631.199911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076631.199913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076631.199917:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3619 00000100:00080000:1.0:1769076631.199919:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076631.199922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076631.199930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076631.199942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612626688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076631.199950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076631.199953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076631.199955:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076631.199958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3619 00000100:00080000:1.0:1769076631.199961:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076631.199964:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076631.199967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076631.199975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612626816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076631.199984:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612626560:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076631.200005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076631.200009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076631.200010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076631.200014:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3619 00000100:00080000:1.0:1769076631.200018:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076631.200020:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076631.200024:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076631.200034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612626944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076631.200102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076631.200152:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076631.200187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076632.095747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076632.095810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076633.119725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076633.119872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076634.143759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076634.143840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076634.272231:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076634.272308:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076634.272335:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076634.272394:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076634.272406:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076634.272420:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769076634.272453:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076634] req@ffff8b1006136d80 x1855007612626816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076634.272490:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612626816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076634.272506:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076634.272514:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076634.272521:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076634.272575:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076634] req@ffff8b1006136680 x1855007612626944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076634.272602:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612626944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076634.272629:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076634.272635:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076634.272708:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076634.272725:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076634] req@ffff8b1006137100 x1855007612626560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076634.272745:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612626560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076634.272771:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076634.272777:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076634.272782:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612626560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076634.272793:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076634] req@ffff8b1006134700 x1855007612626688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076634.272811:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612626688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076634.272823:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076634.272830:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076634.272836:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612626688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076635.167721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076635.167861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076636.191814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076636.191900:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076636.319850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.319867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076636.319874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.319890:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3624 00000100:00080000:3.0:1769076636.319908:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.319919:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076636.319979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612627072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076636.320025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.320031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076636.320034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.320039:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3624 00000100:00080000:3.0:1769076636.320044:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.320049:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076636.320062:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076636.320077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612627200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076636.320091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.320096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076636.320098:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.320103:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3624 00000100:00080000:3.0:1769076636.320107:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.320111:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076636.320118:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076636.320133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612627328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076636.320148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.320153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076636.320155:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.320163:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3624 00000100:00080000:3.0:1769076636.320167:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076636.320174:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612627072:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076636.320175:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076636.320183:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076636.320215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612627456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076636.320327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076636.320411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff8b10037dc000 x1855007612627072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076636.320452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612627072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076636.320474:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076636.320483:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076636.320542:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612627072:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076636.320560:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627200:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076636.320607:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076636.320633:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff8b10037dea00 x1855007612627200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076636.320740:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612627200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076636.320758:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076636.320765:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076636.320772:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627200:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076636.320823:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076636.320874:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076636.320898:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff8b10037dd880 x1855007612627328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076636.320924:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612627328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076636.320942:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076636.320948:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076636.320955:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076636.320968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076636.321020:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076636.321047:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff8b10037df100 x1855007612627456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076636.321073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612627456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076636.321091:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076636.321098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076636.321105:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076637.215740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076637.215848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076638.239755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076638.239935:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076639.263759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076639.263937:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076640.287752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076640.287957:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076641.311767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076641.311979:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076641.439858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076641.439874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076641.439882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076641.439899:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3629 00000100:00080000:2.0:1769076641.439919:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076641.439930:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076641.440026:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612627584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076641.440117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076641.440128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076641.440134:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076641.440145:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3629 00000100:00080000:2.0:1769076641.440160:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076641.440171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076641.440192:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076641.440265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612627712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076641.440291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076641.440298:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076641.440319:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076641.440329:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3629 00000100:00080000:2.0:1769076641.440339:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076641.440342:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612627584:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076641.440354:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076641.440390:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076641.440436:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612627840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076641.440448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076641.440470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076641.440472:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076641.440478:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3629 00000100:00080000:2.0:1769076641.440496:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076641.440503:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076641.440511:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076641.440522:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076641.440531:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612627968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076641.440822:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff8b1005d59880 x1855007612627584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076641.440853:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612627584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076641.440955:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076641.440977:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076641.440992:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612627584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076641.441031:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627712:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076641.441071:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076641.441092:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff8b1005d59500 x1855007612627712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076641.441109:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612627712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076641.441121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076641.441125:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076641.441130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627712:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076641.441137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076641.441161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076641.441173:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff8b1005d5ad80 x1855007612627840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076641.441189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612627840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076641.441199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076641.441203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076641.441207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076641.441214:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076641.441236:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076641.441249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff8b1005d58380 x1855007612627968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076641.441264:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612627968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076641.441274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076641.441278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076641.441281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612627968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076642.335781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076642.335953:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076643.359750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076643.359859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076644.383792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076644.383952:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076645.407783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076645.407871:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076646.431715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076646.431808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076646.559784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076646.559796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076646.559802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076646.559814:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3634 00000100:00080000:2.0:1769076646.559827:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076646.559834:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076646.559878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612628096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076646.559906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076646.559909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076646.559911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076646.559914:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3634 00000100:00080000:2.0:1769076646.559916:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076646.559919:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076646.559929:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076646.559938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612628224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076646.559945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076646.559948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076646.559949:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076646.559952:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3634 00000100:00080000:2.0:1769076646.559953:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076646.559956:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076646.559959:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076646.559970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612628352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076646.559982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076646.559985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076646.559986:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076646.559991:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612628096:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076646.559992:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3634 00000100:00080000:2.0:1769076646.559997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076646.560003:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076646.560007:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076646.560015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612628480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076646.560078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076646.560126:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076646] req@ffff8b1005d5b100 x1855007612628096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076646.560148:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612628096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076646.560160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076646.560165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076646.560169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612628096:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076646.560177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628224:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076646.560202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076646.560216:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076646] req@ffff8b1005d5aa00 x1855007612628224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076646.560230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612628224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076646.560238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076646.560242:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076646.560246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628224:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076646.560277:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076646.560300:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076646.560310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076646] req@ffff8b1005d58380 x1855007612628352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076646.560336:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612628352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076646.560343:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076646.560345:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076646.560347:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076646.560351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076646.560366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076646.560374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076646] req@ffff8b1005d5ad80 x1855007612628480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076646.560382:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612628480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076646.560388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076646.560391:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076646.560393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076647.455773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076647.455833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076648.479750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076648.479888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076649.503727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076649.503783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076650.527707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076650.527771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076651.551727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076651.551956:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076651.679815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076651.679826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076651.679831:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076651.679853:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3639 00000100:00080000:1.0:1769076651.679867:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076651.679875:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076651.679931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612628608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076651.679966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076651.679971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076651.679973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076651.679979:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3639 00000100:00080000:1.0:1769076651.679983:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076651.679989:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076651.680013:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076651.680032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612628736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076651.680045:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612628608:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076651.680047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076651.680051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076651.680053:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076651.680057:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3639 00000100:00080000:1.0:1769076651.680071:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076651.680083:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076651.680089:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076651.680106:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612628864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076651.680113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076651.680116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076651.680118:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076651.680121:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3639 00000100:00080000:1.0:1769076651.680124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076651.680127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076651.680132:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076651.680142:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612628992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076651.680156:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076651.680198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff8b1006137480 x1855007612628608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076651.680218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612628608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076651.680228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076651.680249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076651.680253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612628608:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076651.680296:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628736:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076651.680315:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076651.680325:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff8b1006134a80 x1855007612628736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076651.680333:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612628736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076651.680340:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076651.680343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076651.680345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628736:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076651.680349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076651.680361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076651.680368:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff8b1006134e00 x1855007612628864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076651.680375:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612628864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076651.680381:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076651.680383:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076651.680385:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076651.680389:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076651.680400:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076651.680406:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff8b1006134700 x1855007612628992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076651.680413:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612628992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076651.680418:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076651.680420:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076651.680422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612628992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076652.575729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076652.575790:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076653.599727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076653.599800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076654.623748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076654.623826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076655.647739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076655.647819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076656.671724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076656.671827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076656.799764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076656.799775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076656.799779:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076656.799790:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3644 00000100:00080000:0.0:1769076656.799801:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076656.799809:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076656.799847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612629120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076656.799875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076656.799878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076656.799881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076656.799885:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3644 00000100:00080000:0.0:1769076656.799888:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076656.799894:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076656.799904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076656.799915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612629120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076656.799921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612629248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076656.799934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076656.799938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076656.799941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076656.799945:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3644 00000100:00080000:0.0:1769076656.799949:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076656.799954:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076656.799959:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076656.799970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612629376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076656.799976:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076656.799993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076656.799996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076656.799998:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076656.800004:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3644 00000100:00080000:0.0:1769076656.800009:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076656.800009:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff8b1006024a80 x1855007612629120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.800019:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076656.800026:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076656.800031:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612629120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076656.800041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076656.800043:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612629504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076656.800046:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076656.800054:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612629120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076656.800080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629248:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076656.800098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076656.800107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff8b1006025f80 x1855007612629248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076656.800115:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612629248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076656.800120:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076656.800123:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076656.800126:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076656.800130:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076656.800141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076656.800164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff8b1006027b80 x1855007612629376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076656.800174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612629376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076656.800179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076656.800181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076656.800183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076656.800187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076656.800202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076656.800210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff8b1006026d80 x1855007612629504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076656.800217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612629504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076656.800222:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076656.800224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076656.800226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076657.695746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076657.695815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076658.719704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076658.719791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076659.743703:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076659.743788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076660.767762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076660.767849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076661.791715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076661.791797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076661.919749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076661.919758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076661.919762:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076661.919770:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3650 00000100:00080000:0.0:1769076661.919780:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076661.919786:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076661.919820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612629632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076661.919845:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076661.919848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076661.919849:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076661.919852:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3650 00000100:00080000:0.0:1769076661.919854:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076661.919856:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076661.919865:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076661.919877:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612629760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076661.919887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076661.919891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076661.919894:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076661.919898:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3650 00000100:00080000:0.0:1769076661.919903:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076661.919907:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076661.919914:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076661.919923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612629632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076661.919927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612629888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076661.919936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076661.919939:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076661.919941:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076661.919944:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3650 00000100:00080000:0.0:1769076661.919947:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076661.919950:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076661.919954:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076661.919962:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612630016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076661.919986:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076661.920030:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff8b1006026300 x1855007612629632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076661.920051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612629632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076661.920080:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076661.920086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076661.920090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612629632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076661.920097:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629760:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076661.920122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076661.920133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff8b1006024000 x1855007612629760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076661.920142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612629760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076661.920148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076661.920150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076661.920153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629760:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076661.920171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076661.920186:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076661.920193:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff8b1006025880 x1855007612629888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076661.920201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612629888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076661.920206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076661.920208:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076661.920210:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612629888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076661.920213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076661.920224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076661.920230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff8b1006026a00 x1855007612630016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076661.920237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612630016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076661.920242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076661.920244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076661.920246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076662.815760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076662.815854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076663.839719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076663.839811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076664.863744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076664.863834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076665.887747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076665.887825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076666.911750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076666.911821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076667.039737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076667.039746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076667.039750:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076667.039758:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3655 00000100:00080000:0.0:1769076667.039768:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076667.039774:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076667.039805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612630144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076667.039828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076667.039831:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076667.039832:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076667.039835:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3655 00000100:00080000:0.0:1769076667.039837:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076667.039840:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076667.039847:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076667.039855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612630272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076667.039861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076667.039864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076667.039865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076667.039867:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3655 00000100:00080000:0.0:1769076667.039869:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076667.039872:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076667.039875:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076667.039882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612630400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076667.039889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076667.039892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076667.039893:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076667.039899:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3655 00000100:00100000:3.0:1769076667.039899:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612630144:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076667.039902:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076667.039908:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076667.039912:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076667.039921:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612630528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076667.039962:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076667.040018:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff8b1006025180 x1855007612630144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076667.040033:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612630144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076667.040041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076667.040045:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076667.040048:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612630144:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076667.040053:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630272:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076667.040067:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076667.040076:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff8b1006027800 x1855007612630272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076667.040084:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612630272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076667.040089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076667.040091:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076667.040093:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630272:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076667.040113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076667.040124:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076667.040131:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff8b1006027100 x1855007612630400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076667.040143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612630400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076667.040148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076667.040150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076667.040152:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076667.040156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076667.040166:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076667.040172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076667/real 1769076667] req@ffff8b1006026680 x1855007612630528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076667.040179:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612630528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076667.040184:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076667.040186:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076667.040188:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076667.935729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076667.935834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076668.959712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076668.959766:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076669.983722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076669.983787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076671.007740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076671.007806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076672.031767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076672.031852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076672.159817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.159826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076672.159831:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.159839:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3660 00000100:00080000:2.0:1769076672.159849:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.159855:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076672.159928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612630656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.159948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.159951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076672.159953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.159955:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3660 00000100:00080000:2.0:1769076672.159958:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.159961:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076672.159969:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076672.159979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612630784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.159989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.159991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076672.159993:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076672.159994:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612630656:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076672.159996:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3660 00000100:00080000:2.0:1769076672.160001:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.160005:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076672.160008:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076672.160032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612630912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.160039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.160042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076672.160043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.160047:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3660 00000100:00080000:2.0:1769076672.160050:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.160053:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076672.160057:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076672.160065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612631040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076672.160085:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076672.160125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff8b1005d59500 x1855007612630656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076672.160142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612630656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076672.160151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076672.160154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076672.160158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612630656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076672.160178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630784:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076672.160199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076672.160214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff8b1005d59880 x1855007612630784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076672.160245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612630784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076672.160254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076672.160258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076672.160262:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076672.160269:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076672.160290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076672.160318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff8b1005d5b480 x1855007612630912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076672.160328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612630912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076672.160340:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076672.160343:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076672.160345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612630912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076672.160350:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076672.160363:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076672.160387:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff8b1005d58e00 x1855007612631040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076672.160401:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612631040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076672.160411:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076672.160416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076672.160420:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076673.055763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076673.055996:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076674.079841:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076674.079975:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076675.103809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076675.103931:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076676.127887:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076676.127929:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076677.151854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076677.151950:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076677.279758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.279771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076677.279776:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.279787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3665 00000100:00080000:2.0:1769076677.279800:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.279854:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076677.279890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612631168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.279943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.279949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076677.279951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.279957:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3665 00000100:00080000:2.0:1769076677.279964:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.279970:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076677.279983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076677.280005:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612631296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.280024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076677.280026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612631168:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076677.280030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076677.280034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.280042:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3665 00000100:00080000:2.0:1769076677.280048:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.280066:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076677.280076:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076677.280097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612631424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.280114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.280119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076677.280123:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769076677.280130:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076677.280137:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3665 00000100:00080000:2.0:1769076677.280144:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.280150:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076677.280156:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076677.280172:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612631552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076677.280201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff8b1005d5a300 x1855007612631168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076677.280231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612631168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076677.280245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076677.280251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076677.280257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612631168:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076677.280284:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631296:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076677.280318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076677.280338:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff8b1005d5bb80 x1855007612631296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076677.280357:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612631296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076677.280365:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076677.280368:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076677.280371:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631296:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076677.280377:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076677.280399:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076677.280410:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff8b1005d58a80 x1855007612631424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076677.280422:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612631424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076677.280436:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076677.280440:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076677.280444:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076677.280449:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076677.280466:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076677.280495:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff8b1005d5b800 x1855007612631552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076677.280510:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612631552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076677.280522:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076677.280528:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076677.280533:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076678.175754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076678.175849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076679.199757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076679.199847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076680.223810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076680.223966:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076681.247762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076681.247855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076682.271780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076682.271879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076682.399752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076682.399762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076682.399766:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076682.399786:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3670 00000100:00080000:2.0:1769076682.399797:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076682.399804:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076682.399853:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612631680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076682.399893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076682.399899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076682.399901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076682.399906:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3670 00000100:00080000:2.0:1769076682.399910:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076682.399915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076682.399926:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076682.399944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612631808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076682.399975:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612631680:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076682.399976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076682.399988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076682.399990:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076682.399995:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3670 00000100:00080000:2.0:1769076682.400000:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076682.400004:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076682.400010:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076682.400044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612631936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076682.400061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076682.400064:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076682.400067:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076682.400071:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3670 00000100:00080000:2.0:1769076682.400076:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076682.400082:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076682.400102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076682.400119:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076682.400130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612632064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076682.400157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff8b1005d59180 x1855007612631680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076682.400176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612631680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076682.400186:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076682.400190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076682.400194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612631680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076682.400217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631808:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076682.400238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076682.400249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff8b1005d58000 x1855007612631808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076682.400277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612631808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076682.400283:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076682.400286:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076682.400289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076682.400294:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076682.400320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076682.400329:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff8b1005d5b800 x1855007612631936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076682.400339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612631936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076682.400346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076682.400349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076682.400362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612631936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076682.400367:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076682.400381:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076682.400389:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff8b1005d58a80 x1855007612632064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076682.400399:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612632064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076682.400405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076682.400407:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076682.400410:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076683.295779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076683.295918:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076684.319788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076684.319898:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076685.343776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076685.343851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076686.367762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076686.367839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076687.391792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076687.391873:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076687.519842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076687.519855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076687.519862:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076687.519875:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3675 00000100:00080000:0.0:1769076687.519892:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076687.519901:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076687.519990:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612632192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076687.520031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076687.520038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076687.520041:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076687.520047:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3675 00000100:00080000:0.0:1769076687.520052:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076687.520058:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076687.520071:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076687.520092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612632320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076687.520092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612632192:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076687.520127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076687.520132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076687.520135:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076687.520142:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3675 00000100:00080000:0.0:1769076687.520148:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076687.520155:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076687.520164:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076687.520186:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612632448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076687.520193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076687.520207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076687.520212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076687.520216:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076687.520224:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3675 00000100:00080000:0.0:1769076687.520231:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076687.520238:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076687.520247:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076687.520251:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff8b1006026680 x1855007612632192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076687.520272:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612632576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076687.520278:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612632192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076687.520317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076687.520325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076687.520330:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612632192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076687.520360:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632320:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076687.520382:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076687.520394:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff8b1006027100 x1855007612632320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076687.520407:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612632320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076687.520422:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076687.520425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076687.520428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076687.520433:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076687.520449:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076687.520459:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff8b1006027800 x1855007612632448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076687.520495:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612632448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076687.520508:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076687.520513:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076687.520518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076687.520528:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076687.520561:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076687.520580:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff8b1006025180 x1855007612632576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076687.520594:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612632576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076687.520602:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076687.520606:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076687.520610:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076688.415757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076688.415857:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076689.439752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076689.439955:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076690.463735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076690.463807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076691.487783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076691.487938:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076692.511800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076692.511969:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076692.639797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076692.639814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076692.639823:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076692.639842:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3680 00000100:00080000:3.0:1769076692.639861:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076692.639874:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076692.639931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612632704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076692.639974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076692.639980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076692.639983:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076692.639990:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3680 00000100:00080000:3.0:1769076692.639997:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076692.640003:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076692.640016:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076692.640028:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612632704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076692.640044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612632832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076692.640062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076692.640069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076692.640073:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076692.640079:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3680 00000100:00080000:3.0:1769076692.640083:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076692.640089:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076692.640098:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076692.640111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612632960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076692.640123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076692.640127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076692.640129:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076692.640134:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3680 00000800:00000100:1.0:1769076692.640146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076692.640150:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076692.640158:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076692.640167:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076692.640183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612633088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076692.640203:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff8b10037ddc00 x1855007612632704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076692.640231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612632704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076692.640248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076692.640257:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076692.640263:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612632704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076692.640296:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632832:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076692.640321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076692.640331:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff8b10037dce00 x1855007612632832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076692.640341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612632832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076692.640348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076692.640351:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076692.640353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632832:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076692.640359:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076692.640373:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076692.640381:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff8b10037df480 x1855007612632960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076692.640389:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612632960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076692.640395:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076692.640397:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076692.640399:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612632960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076692.640403:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076692.640416:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076692.640423:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff8b10037df800 x1855007612633088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076692.640431:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612633088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076692.640436:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076692.640438:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076692.640440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076693.535826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076693.536036:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076694.559764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076694.559884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076695.583754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076695.583882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076696.607784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076696.607949:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076697.631783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076697.631958:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076697.759726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076697.759735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076697.759739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076697.759746:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3685 00000100:00080000:2.0:1769076697.759756:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076697.759763:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076697.759802:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612633216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076697.759832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076697.759836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076697.759838:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076697.759843:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3685 00000100:00080000:2.0:1769076697.759847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076697.759852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076697.759862:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076697.759874:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612633344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076697.759882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076697.759885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076697.759888:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076697.759891:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3685 00000100:00080000:2.0:1769076697.759893:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076697.759904:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076697.759909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076697.759919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612633472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076697.759920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612633216:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076697.759947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076697.759951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076697.759955:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076697.759961:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3685 00000100:00080000:2.0:1769076697.759966:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076697.759971:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076697.759978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076697.759994:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612633600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076697.760057:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076697.760101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076697.760127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076698.655711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076698.655821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076699.679776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076699.679878:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076700.703789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076700.703931:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769076700.831988:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769076700.832052:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076700.832081:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076700.832147:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076700.832161:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769076700.832193:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076700.832244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076700] req@ffff8b1005d58e00 x1855007612633472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076700.832295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612633472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076700.832316:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076700.832325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076700.832333:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076700.832347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076700] req@ffff8b1005d5b480 x1855007612633600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076700.832400:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612633600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076700.832460:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076700.832467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076700.832474:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076700.832494:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076700] req@ffff8b1005d5bb80 x1855007612633216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076700.832516:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612633216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076700.832530:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076700.832535:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076700.832541:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612633216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076700.832553:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076700] req@ffff8b1005d5a300 x1855007612633344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076700.832574:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612633344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076700.832587:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076700.832594:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076700.832599:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076701.727772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076701.727944:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076702.751770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076702.751935:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076702.879871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076702.879888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076702.879893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076702.879905:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3690 00000100:00080000:3.0:1769076702.879919:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076702.879929:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076702.879972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612633728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076702.880010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076702.880015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076702.880018:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076702.880023:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3690 00000100:00080000:3.0:1769076702.880028:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076702.880036:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076702.880051:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076702.880081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612633856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076702.880112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612633728:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076702.880131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076702.880141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076702.880145:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076702.880157:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3690 00000100:00080000:3.0:1769076702.880165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076702.880174:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076702.880185:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076702.880229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076702.880230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612633984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076702.880256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076702.880264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076702.880269:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076702.880278:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3690 00000100:00080000:3.0:1769076702.880288:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076702.880298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff8b10037dc700 x1855007612633728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076702.880307:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076702.880323:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076702.880351:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612634112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076702.880403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612633728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076702.880440:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076702.880451:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076702.880461:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612633728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076702.880502:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633856:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076702.880579:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076702.880610:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff8b10037df800 x1855007612633856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076702.880696:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612633856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076702.880717:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076702.880725:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076702.880733:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633856:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076702.880749:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076702.880798:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076702.880827:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff8b10037df480 x1855007612633984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076702.880857:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612633984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076702.880876:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076702.880883:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076702.880891:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612633984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076702.880905:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076702.880948:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076702.880973:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff8b10037dce00 x1855007612634112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076702.881000:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612634112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076702.881029:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076702.881037:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076702.881045:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076703.775802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076703.775968:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076704.799739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076704.799868:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076705.823776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076705.823926:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076706.847741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076706.847829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076707.871704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076707.871778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076707.999776:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076707.999784:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076707.999788:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076707.999796:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3696 00000100:00080000:1.0:1769076707.999805:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076707.999811:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076707.999844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612634240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076707.999869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076707.999872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076707.999873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076707.999876:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3696 00000100:00080000:1.0:1769076707.999879:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076707.999882:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076707.999889:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076707.999899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612634368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076707.999904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612634240:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076707.999910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076707.999914:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076707.999915:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076707.999918:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3696 00000100:00080000:1.0:1769076707.999921:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076707.999924:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076707.999928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076707.999938:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612634496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076707.999944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076707.999947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076707.999948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076707.999950:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3696 00000100:00080000:1.0:1769076707.999960:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076707.999963:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769076707.999967:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769076707.999968:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076707.999979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612634624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076708.000002:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff8b1006137100 x1855007612634240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076708.000016:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612634240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076708.000025:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076708.000028:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076708.000031:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612634240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076708.000049:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634368:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076708.000064:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076708.000072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076708] req@ffff8b1006136680 x1855007612634368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076708.000080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612634368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076708.000086:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076708.000088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076708.000090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634368:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076708.000093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076708.000105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076708.000111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076708] req@ffff8b1006136d80 x1855007612634496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076708.000124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612634496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076708.000129:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076708.000131:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076708.000133:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076708.000136:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076708.000154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076708.000162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076708] req@ffff8b1006135500 x1855007612634624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076708.000169:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612634624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076708.000175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076708.000177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076708.000179:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076708.895745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076708.895847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076709.919737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076709.919792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076710.943722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076710.943803:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076711.967737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076711.967884:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076712.991713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076712.991796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076713.119775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076713.119788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076713.119793:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076713.119804:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3701 00000100:00080000:1.0:1769076713.119817:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076713.119826:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076713.119864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612634752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076713.119894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076713.119898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076713.119902:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076713.119909:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3701 00000100:00080000:1.0:1769076713.119914:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076713.119920:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076713.119931:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076713.119938:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612634752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076713.119951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612634880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076713.119965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076713.119969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076713.119972:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076713.119978:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3701 00000100:00080000:1.0:1769076713.119983:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076713.119988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076713.119995:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076713.120005:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076713.120013:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612635008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076713.120025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076713.120028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076713.120040:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076713.120049:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3701 00000100:00100000:2.0:1769076713.120049:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff8b1006135880 x1855007612634752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076713.120057:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076713.120064:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076713.120070:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612634752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076713.120071:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1769076713.120084:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076713.120090:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076713.120093:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612635136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076713.120094:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612634752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076713.120127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634880:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076713.120146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076713.120155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff8b1006137b80 x1855007612634880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076713.120164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612634880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076713.120170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076713.120172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076713.120174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612634880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076713.120178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076713.120191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076713.120198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff8b1006135c00 x1855007612635008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076713.120224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612635008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076713.120235:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076713.120237:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076713.120239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076713.120244:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076713.120258:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076713.120265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076713] req@ffff8b1006136300 x1855007612635136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076713.120276:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612635136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076713.120285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076713.120288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076713.120291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076714.015727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076714.015835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076715.039696:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076715.039830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076716.063740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076716.063879:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076717.087751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076717.087863:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076718.111746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076718.111866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076718.239761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076718.239775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076718.239781:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076718.239793:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3706 00000100:00080000:1.0:1769076718.239808:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076718.239817:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076718.239864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612635264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076718.239903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076718.239908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076718.239910:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076718.239916:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3706 00000100:00080000:1.0:1769076718.239920:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076718.239926:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076718.239939:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076718.239954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612635392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076718.239969:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076718.239974:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076718.239977:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076718.239987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612635264:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076718.239988:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3706 00000100:00080000:1.0:1769076718.239999:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076718.240008:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076718.240015:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076718.240033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612635520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076718.240047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076718.240051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076718.240054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076718.240058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3706 00000100:00080000:1.0:1769076718.240063:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076718.240068:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076718.240075:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076718.240088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076718.240096:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612635648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076718.240180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff8b1006136a00 x1855007612635264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076718.240209:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612635264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076718.240224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076718.240230:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076718.240235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612635264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076718.240273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635392:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076718.240323:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076718.240350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff8b1006135180 x1855007612635392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076718.240377:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612635392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076718.240388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076718.240393:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076718.240397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635392:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076718.240406:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076718.240435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076718.240449:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff8b1006137800 x1855007612635520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076718.240475:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612635520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076718.240486:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076718.240489:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076718.240493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076718.240500:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076718.240520:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076718.240532:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff8b1006136300 x1855007612635648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076718.240548:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612635648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076718.240558:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076718.240562:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076718.240565:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076719.135699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076719.135781:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076720.159771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076720.159892:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076721.183777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076721.183920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076722.207750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076722.207851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076723.231729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076723.231813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076723.359745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076723.359754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076723.359758:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076723.359765:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3711 00000100:00080000:3.0:1769076723.359775:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076723.359798:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076723.359846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612635776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076723.359884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076723.359890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076723.359892:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076723.359897:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3711 00000100:00080000:3.0:1769076723.359902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076723.359907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076723.359918:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076723.359932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612635904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076723.359955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076723.359960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076723.359963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076723.359968:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3711 00000100:00080000:3.0:1769076723.359972:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076723.359977:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076723.359983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076723.359997:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612636032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076723.360009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076723.360013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076723.360016:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076723.360022:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3711 00000100:00080000:3.0:1769076723.360071:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076723.360073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612635776:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076723.360079:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076723.360086:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076723.360101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612636160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076723.360161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076723.360221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff8b10037ddc00 x1855007612635776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076723.360246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612635776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076723.360260:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076723.360265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076723.360282:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612635776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076723.360292:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635904:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076723.360318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076723.360333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff8b10037df100 x1855007612635904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076723.360348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612635904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076723.360375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076723.360379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076723.360383:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612635904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076723.360390:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076723.360413:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076723.360424:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff8b10037dd880 x1855007612636032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076723.360439:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612636032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076723.360449:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076723.360452:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076723.360456:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076723.360485:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076723.360525:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076723.360532:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff8b10037dea00 x1855007612636160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076723.360539:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612636160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076723.360544:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076723.360546:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076723.360549:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076724.255896:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076724.256073:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076725.279805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076725.279895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076726.303779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076726.303844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076727.327724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076727.327805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076728.351800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076728.351922:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076728.479861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076728.479874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076728.479880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076728.479892:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3716 00000100:00080000:0.0:1769076728.479907:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076728.479916:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076728.480015:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612636288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076728.480051:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076728.480056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076728.480059:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076728.480064:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3716 00000100:00080000:0.0:1769076728.480069:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076728.480073:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076728.480104:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076728.480120:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612636416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076728.480135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076728.480140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076728.480142:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076728.480153:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3716 00000100:00080000:0.0:1769076728.480158:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076728.480162:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076728.480174:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612636288:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076728.480175:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076728.480196:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612636544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076728.480213:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076728.480218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076728.480220:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076728.480225:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3716 00000100:00080000:0.0:1769076728.480230:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076728.480234:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076728.480240:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076728.480255:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612636672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076728.480272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076728.480323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff8b1006026a00 x1855007612636288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076728.480348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612636288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076728.480361:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076728.480367:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076728.480373:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612636288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076728.480382:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636416:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076728.480407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076728.480422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff8b1006025880 x1855007612636416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076728.480438:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612636416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076728.480449:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076728.480452:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076728.480457:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636416:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076728.480486:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076728.480507:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076728.480520:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff8b1006024000 x1855007612636544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076728.480535:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612636544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076728.480545:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076728.480549:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076728.480552:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076728.480559:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076728.480579:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076728.480590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff8b1006026300 x1855007612636672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076728.480605:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612636672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076728.480615:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076728.480618:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076728.480622:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076729.375794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076729.375954:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076730.399768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076730.399940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076731.423775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076731.423887:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076732.447700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076732.447928:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076733.471740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076733.471841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076733.599763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076733.599772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076733.599776:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076733.599785:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3721 00000100:00080000:2.0:1769076733.599795:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076733.599800:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076733.599841:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612636800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076733.599889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076733.599892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076733.599893:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076733.599896:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3721 00000100:00080000:2.0:1769076733.599899:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076733.599902:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076733.599909:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076733.599920:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612636928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076733.599928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076733.599931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076733.599932:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076733.599937:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612636800:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076733.599938:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3721 00000100:00080000:2.0:1769076733.599943:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076733.599947:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076733.599950:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076733.599960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612637056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076733.599966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076733.599969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076733.599970:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076733.599973:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3721 00000100:00080000:2.0:1769076733.599987:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076733.599990:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076733.599994:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076733.599999:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076733.600008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612637184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076733.600036:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff8b1005d59880 x1855007612636800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076733.600051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612636800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076733.600059:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076733.600062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076733.600065:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612636800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076733.600083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636928:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076733.600097:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076733.600105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff8b1005d59500 x1855007612636928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076733.600113:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612636928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076733.600119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076733.600121:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076733.600123:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612636928:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076733.600127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076733.600138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076733.600144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff8b1005d5ad80 x1855007612637056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076733.600156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612637056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076733.600161:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076733.600163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076733.600165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076733.600169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076733.600179:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076733.600185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff8b1005d58380 x1855007612637184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076733.600192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612637184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076733.600197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076733.600199:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076733.600201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076734.495767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076734.495905:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076735.519743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076735.519860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076736.543709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076736.543921:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076737.567725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076737.567846:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076738.591755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076738.591865:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076738.719758:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.719766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076738.719787:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.719796:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3726 00000100:00080000:2.0:1769076738.719806:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.719811:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076738.719861:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612637312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.719892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.719896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076738.719898:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.719913:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3726 00000100:00080000:2.0:1769076738.719918:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.719922:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076738.719931:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076738.719945:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612637440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.719952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.719956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076738.719957:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.719960:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3726 00000100:00080000:2.0:1769076738.719963:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.719967:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076738.719970:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612637312:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769076738.719972:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076738.719986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612637568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.719994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.719996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076738.719997:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.719999:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3726 00000100:00080000:2.0:1769076738.720002:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.720005:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076738.720008:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076738.720015:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612637696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076738.720031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076738.720063:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff8b1005d5aa00 x1855007612637312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076738.720077:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612637312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076738.720085:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076738.720088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076738.720091:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612637312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076738.720096:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637440:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076738.720110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076738.720118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff8b1005d5b100 x1855007612637440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076738.720126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612637440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076738.720131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076738.720133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076738.720135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637440:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076738.720153:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076738.720164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076738.720170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff8b1005d58380 x1855007612637568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076738.720178:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612637568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076738.720188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076738.720190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076738.720192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076738.720196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076738.720206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076738.720212:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff8b1005d5ad80 x1855007612637696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076738.720219:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612637696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076738.720224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076738.720226:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076738.720228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076739.615719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076739.615808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076740.639786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076740.639903:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076741.663854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076741.663997:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076742.687770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076742.687831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076743.711719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076743.711784:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076743.839811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076743.839832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076743.839835:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076743.839844:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3731 00000100:00080000:0.0:1769076743.839854:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076743.839860:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076743.839892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612637824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076743.839916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076743.839920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076743.839922:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076743.839927:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3731 00000100:00080000:0.0:1769076743.839930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076743.839935:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076743.839944:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076743.839956:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612637952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076743.839966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076743.839970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076743.839971:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076743.839971:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612637824:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076743.839975:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3731 00000100:00080000:0.0:1769076743.839979:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076743.839982:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076743.839987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076743.839997:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612638080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076743.840005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076743.840008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076743.840010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076743.840013:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3731 00000100:00080000:0.0:1769076743.840015:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076743.840019:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076743.840023:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076743.840042:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612638208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076743.840043:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076743.840078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff8b1006026d80 x1855007612637824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076743.840109:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612637824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076743.840119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076743.840124:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076743.840128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612637824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076743.840151:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637952:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076743.840170:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076743.840180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff8b1006027b80 x1855007612637952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076743.840188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612637952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076743.840204:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076743.840207:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076743.840209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612637952:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076743.840212:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076743.840224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076743.840230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff8b1006025f80 x1855007612638080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076743.840238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612638080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076743.840243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076743.840245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076743.840247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076743.840251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076743.840261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076743.840267:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff8b1006024a80 x1855007612638208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076743.840274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612638208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076743.840280:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076743.840282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076743.840284:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076744.735775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076744.735834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076745.759769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076745.759830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076746.783743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076746.783861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076747.807758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076747.807823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076748.831729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076748.831792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076748.959778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.959787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076748.959792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.959803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3737 00000100:00080000:1.0:1769076748.959814:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.959822:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076748.959855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612638336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.959886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.959890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076748.959894:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.959899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3737 00000100:00080000:1.0:1769076748.959903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.959910:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076748.959913:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612638336:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076748.959921:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076748.959940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612638464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.959952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.959956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076748.959958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.959963:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3737 00000100:00080000:1.0:1769076748.959967:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.959974:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769076748.959975:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769076748.959983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076748.959998:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612638592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.960008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076748.960013:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff8b1006135c00 x1855007612638336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076748.960014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076748.960019:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.960028:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3737 00000100:00080000:1.0:1769076748.960032:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076748.960035:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612638336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076748.960040:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:0.0:1769076748.960047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076748.960050:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076748.960053:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076748.960059:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612638336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076748.960066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612638720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076748.960081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638464:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076748.960112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076748.960122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff8b1006137b80 x1855007612638464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076748.960131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612638464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076748.960137:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076748.960139:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076748.960141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076748.960145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076748.960158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076748.960165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff8b1006135880 x1855007612638592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076748.960172:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612638592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076748.960177:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076748.960179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076748.960181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076748.960191:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076748.960202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076748.960209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff8b1006135500 x1855007612638720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076748.960218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612638720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076748.960224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076748.960226:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076748.960228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076749.855749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076749.855804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076750.879695:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076750.879769:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076751.903718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076751.903789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076752.927754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076752.927823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076753.951704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076753.951786:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076754.079744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076754.079756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076754.079761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076754.079771:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3742 00000100:00080000:2.0:1769076754.079783:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076754.079791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076754.079828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612638848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076754.079860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076754.079864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076754.079867:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076754.079871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3742 00000100:00080000:2.0:1769076754.079875:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076754.079879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076754.079889:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076754.079903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612638976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076754.079911:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612638848:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076754.079918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076754.079924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076754.079925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076754.079930:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3742 00000100:00080000:2.0:1769076754.079935:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076754.079939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076754.079945:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076754.079957:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612639104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076754.079968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076754.079972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076754.079974:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076754.079980:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3742 00000800:00000100:0.0:1769076754.079982:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076754.079985:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076754.079992:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076754.079998:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076754.080012:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612639232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076754.080024:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff8b1005d59500 x1855007612638848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.080047:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612638848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.080057:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.080062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076754.080066:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612638848:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076754.080090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638976:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076754.080114:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076754.080128:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff8b1005d59880 x1855007612638976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.080143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612638976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.080151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.080154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076754.080158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612638976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076754.080164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.080184:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076754.080196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff8b1005d5a300 x1855007612639104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.080208:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612639104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.080217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.080220:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076754.080224:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076754.080231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076754.080251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076754.080263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff8b1005d5bb80 x1855007612639232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076754.080276:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612639232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076754.080285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076754.080288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076754.080291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076754.975721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076754.975805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076755.999761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076755.999875:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076757.023714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076757.023777:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076758.047746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076758.047851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076759.071721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076759.071775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076759.199817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076759.199832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076759.199838:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076759.199853:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3747 00000100:00080000:2.0:1769076759.199871:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076759.199881:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076759.199938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612639360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076759.199978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076759.199984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076759.199987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076759.199992:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3747 00000100:00080000:2.0:1769076759.199997:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076759.200002:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076759.200015:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076759.200034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612639488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076759.200069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076759.200078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076759.200082:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076759.200091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612639360:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076759.200096:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3747 00000100:00080000:2.0:1769076759.200109:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076759.200117:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076759.200127:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076759.200164:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612639616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076759.200179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076759.200185:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076759.200192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076759.200196:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076759.200217:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3747 00000100:00080000:2.0:1769076759.200226:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076759.200228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff8b1005d5b480 x1855007612639360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.200242:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076759.200253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612639360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076759.200254:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769076759.200274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076759.200279:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076759.200284:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612639360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076759.200291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612639744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076759.200309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639488:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076759.200345:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076759.200357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff8b1005d58e00 x1855007612639488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076759.200366:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612639488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076759.200375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076759.200381:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076759.200384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076759.200411:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076759.200425:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076759.200452:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff8b1005d58a80 x1855007612639616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076759.200466:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612639616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076759.200475:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076759.200478:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076759.200482:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076759.200489:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076759.200511:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076759.200523:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff8b1005d5b800 x1855007612639744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076759.200535:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612639744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076759.200543:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076759.200546:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076759.200550:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612639744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076760.095733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076760.095798:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076761.119712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076761.119764:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076762.143735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076762.143824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076763.167749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076763.167813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076764.191753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076764.191798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076764.319707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076764.319717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076764.319722:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076764.319732:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3752 00000100:00080000:2.0:1769076764.319745:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076764.319752:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076764.319798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612639872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076764.319826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076764.319829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076764.319830:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076764.319833:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3752 00000100:00080000:2.0:1769076764.319836:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076764.319840:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076764.319849:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076764.319860:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612640000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076764.319865:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612639872:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076764.319871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076764.319875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076764.319876:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076764.319879:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3752 00000100:00080000:2.0:1769076764.319881:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076764.319883:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076764.319886:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076764.319895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612640128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076764.319901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076764.319903:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076764.319904:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076764.319907:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3752 00000100:00080000:2.0:1769076764.319910:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076764.319913:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076764.319917:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076764.319924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612640256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076764.319984:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076764.320007:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076764.320026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076765.215753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076765.215815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076766.239748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076766.239812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076767.263789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076767.263896:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076767.392043:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076767.392099:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076767.392128:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076767.392191:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076767.392209:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076767.392226:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769076767.392345:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076767] req@ffff8b1005d59180 x1855007612640000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076767.392392:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612640000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076767.392414:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076767.392424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076767.392432:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076767.392447:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076767] req@ffff8b1005d5b800 x1855007612640128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076767.392488:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612640128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076767.392505:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076767.392512:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076767.392520:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076767.392533:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076767] req@ffff8b1005d58a80 x1855007612640256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076767.392555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612640256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076767.392570:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076767.392576:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076767.392583:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076767.392595:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076767] req@ffff8b1005d58000 x1855007612639872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076767.392618:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612639872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076767.392634:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076767.392707:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076767.392715:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612639872:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076768.287776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076768.287851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076769.311749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076769.311835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076769.439949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076769.439990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076769.439993:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076769.440003:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3757 00000100:00080000:3.0:1769076769.440014:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076769.440021:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076769.440076:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612640384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076769.440118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076769.440123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076769.440126:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076769.440132:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3757 00000100:00080000:3.0:1769076769.440137:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076769.440143:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076769.440155:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076769.440170:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612640512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076769.440184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076769.440187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076769.440190:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076769.440205:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3757 00000100:00080000:3.0:1769076769.440209:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076769.440213:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076769.440218:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076769.440222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612640384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076769.440233:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612640640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076769.440248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076769.440252:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076769.440287:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076769.440293:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3757 00000100:00080000:3.0:1769076769.440298:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076769.440303:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076769.440312:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076769.440320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076769.440341:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612640768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076769.440374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff8b10037dc000 x1855007612640384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076769.440400:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612640384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076769.440413:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076769.440418:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076769.440423:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612640384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076769.440434:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640512:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076769.440461:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076769.440479:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff8b10037dd500 x1855007612640512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076769.440494:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612640512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076769.440504:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076769.440508:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076769.440512:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640512:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076769.440539:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076769.440564:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076769.440579:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff8b10037ddf80 x1855007612640640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076769.440595:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612640640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076769.440605:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076769.440609:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076769.440613:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076769.440621:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076769.440687:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076769.440703:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff8b10037de300 x1855007612640768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076769.440718:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612640768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076769.440736:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076769.440741:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076769.440746:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612640768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076770.335802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076770.335927:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076771.359702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076771.359804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076772.383750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076772.383805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076773.407738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076773.407803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076774.431725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076774.431780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076774.559725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.559733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076774.559737:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.559746:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3762 00000100:00080000:3.0:1769076774.559756:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.559762:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076774.559806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612640896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076774.559832:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.559835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076774.559837:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.559842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3762 00000100:00080000:3.0:1769076774.559845:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.559849:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076774.559859:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076774.559873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612641024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076774.559887:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612640896:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076774.559887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.559897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076774.559899:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.559903:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3762 00000100:00080000:3.0:1769076774.559907:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.559910:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076774.559915:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076774.559926:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612641152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076774.559933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.559936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076774.559937:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.559939:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3762 00000100:00080000:3.0:1769076774.559942:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.559944:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076774.559947:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076774.559954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612641280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076774.559960:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076774.560003:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff8b10037dd180 x1855007612640896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076774.560026:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612640896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076774.560045:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076774.560050:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076774.560055:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612640896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076774.560079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641024:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076774.560102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076774.560112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff8b10037ded80 x1855007612641024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076774.560121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612641024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076774.560128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076774.560130:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076774.560132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641024:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076774.560136:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076774.560149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076774.560155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff8b10037de680 x1855007612641152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076774.560163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612641152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076774.560198:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076774.560200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076774.560203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076774.560207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076774.560222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076774.560230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff8b10037dc380 x1855007612641280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076774.560238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612641280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076774.560243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076774.560245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076774.560247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076775.455776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076775.455887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076776.479759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076776.479802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076777.503781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076777.503945:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076778.527788:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076778.527789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076779.551766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076779.551939:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076779.679758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076779.679768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076779.679772:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076779.679780:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3767 00000100:00080000:1.0:1769076779.679790:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076779.679796:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076779.679829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612641408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076779.679853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076779.679857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076779.679858:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076779.679861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3767 00000100:00080000:1.0:1769076779.679864:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076779.679874:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076779.679882:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076779.679894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612641536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076779.679902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076779.679905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076779.679906:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076779.679910:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3767 00000100:00080000:1.0:1769076779.679912:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076779.679918:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612641408:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076779.679918:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076779.679922:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076779.679931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612641664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076779.679939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076779.679942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076779.679943:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076779.679946:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3767 00000100:00080000:1.0:1769076779.679948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076779.679951:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076779.679954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076779.679962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612641792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076779.680006:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076779.680046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076779] req@ffff8b1006136d80 x1855007612641408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076779.680063:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612641408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076779.680073:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076779.680076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076779.680080:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612641408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076779.680105:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641536:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076779.680126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076779.680154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076779] req@ffff8b1006136680 x1855007612641536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076779.680169:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612641536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076779.680175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076779.680178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076779.680180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076779.680201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076779.680215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076779.680222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076779] req@ffff8b1006137100 x1855007612641664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076779.680230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612641664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076779.680236:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076779.680238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076779.680240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076779.680244:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076779.680255:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076779.680262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076779] req@ffff8b1006134700 x1855007612641792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076779.680270:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612641792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076779.680276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076779.680277:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076779.680279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612641792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076780.575764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076780.575833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076781.599785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076781.599893:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076782.623712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076782.623768:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076783.647731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076783.647819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076784.671747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076784.671823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076784.799818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076784.799831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076784.799838:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076784.799862:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3772 00000100:00080000:1.0:1769076784.799896:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076784.799908:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076784.799958:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612641920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076784.800022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076784.800045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076784.800049:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076784.800058:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3772 00000100:00080000:1.0:1769076784.800066:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076784.800073:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076784.800083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612641920:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076784.800090:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076784.800121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612642048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076784.800141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076784.800147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076784.800150:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076784.800156:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3772 00000100:00080000:1.0:1769076784.800165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076784.800170:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076784.800178:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076784.800181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076784.800200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612642176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076784.800215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076784.800220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076784.800222:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076784.800228:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3772 00000100:00080000:1.0:1769076784.800232:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076784.800236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff8b1006134e00 x1855007612641920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076784.800243:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076784.800255:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076784.800292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612642304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076784.800298:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612641920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.800320:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076784.800326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076784.800333:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612641920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076784.800360:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642048:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076784.800385:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076784.800399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff8b1006134a80 x1855007612642048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076784.800412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612642048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.800423:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076784.800428:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076784.800433:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642048:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076784.800458:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076784.800521:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076784.800533:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff8b1006137480 x1855007612642176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076784.800545:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612642176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.800552:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076784.800555:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076784.800558:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076784.800563:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076784.800602:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076784.800614:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff8b1006134700 x1855007612642304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076784.800629:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612642304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.800656:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076784.800659:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076784.800664:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076785.695766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076785.695940:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076786.719755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076786.719888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076787.743783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076787.743879:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076788.767819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076788.767903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076789.791796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076789.791986:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076789.919746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076789.919756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076789.919760:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076789.919768:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3778 00000100:00080000:2.0:1769076789.919776:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076789.919782:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076789.919813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612642432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076789.919866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076789.919871:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076789.919874:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076789.919879:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3778 00000100:00080000:2.0:1769076789.919883:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076789.919905:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076789.919928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076789.919940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612642560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076789.919948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076789.919951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076789.919952:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076789.919955:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3778 00000100:00080000:2.0:1769076789.919958:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076789.919961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076789.919966:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076789.919978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612642688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076789.919979:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612642432:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076789.920007:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076789.920017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076789.920019:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076789.920022:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3778 00000100:00080000:2.0:1769076789.920025:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076789.920028:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076789.920031:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076789.920040:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612642816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076789.920078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076789.920120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff8b1005d58e00 x1855007612642432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076789.920135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612642432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076789.920155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076789.920160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076789.920164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612642432:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076789.920170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642560:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076789.920187:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076789.920196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff8b1005d5b480 x1855007612642560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076789.920205:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612642560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076789.920212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076789.920214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076789.920216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076789.920242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076789.920255:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076789.920261:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff8b1005d5bb80 x1855007612642688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076789.920269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612642688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076789.920275:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076789.920277:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076789.920279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076789.920283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076789.920294:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076789.920300:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff8b1005d5a300 x1855007612642816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076789.920308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612642816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076789.920314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076789.920316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076789.920317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612642816:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769076790.815767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076790.815772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076791.839799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076791.839823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076792.863765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076792.863937:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076793.887720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076793.887762:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076794.911816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076794.912068:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076795.039876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076795.039889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076795.039895:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076795.039907:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3783 00000100:00080000:3.0:1769076795.039922:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076795.039930:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076795.039977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612642944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076795.040024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076795.040029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076795.040031:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076795.040037:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3783 00000100:00080000:3.0:1769076795.040041:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076795.040046:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076795.040058:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076795.040074:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612643072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076795.040087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076795.040092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076795.040094:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076795.040099:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3783 00000100:00080000:3.0:1769076795.040103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076795.040108:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076795.040115:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076795.040126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612643200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076795.040141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076795.040148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076795.040150:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612642944:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076795.040152:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076795.040166:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3783 00000100:00080000:3.0:1769076795.040171:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076795.040175:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076795.040182:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076795.040198:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612643328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076795.040250:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076795.040298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff8b10037dca80 x1855007612642944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076795.040322:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612642944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076795.040336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076795.040342:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076795.040347:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612642944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076795.040357:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643072:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076795.040383:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076795.040397:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff8b10037dc380 x1855007612643072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076795.040412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612643072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076795.040423:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076795.040426:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076795.040430:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643072:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076795.040460:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076795.040481:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076795.040493:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff8b10037de680 x1855007612643200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076795.040508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612643200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076795.040518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076795.040521:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076795.040525:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076795.040532:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076795.040551:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076795.040563:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076795/real 1769076795] req@ffff8b10037ded80 x1855007612643328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076795.040578:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612643328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076795.040588:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076795.040592:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076795.040595:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076795.935775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076795.935929:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076796.959749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076796.959826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076797.983915:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076797.984006:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076799.007800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076799.007909:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076800.031780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076800.031847:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076800.159768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076800.159776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076800.159779:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076800.159787:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3788 00000100:00080000:3.0:1769076800.159797:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076800.159803:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076800.159834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612643456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076800.159859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076800.159861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076800.159867:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076800.159870:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3788 00000100:00080000:3.0:1769076800.159872:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076800.159875:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076800.159882:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076800.159892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612643584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076800.159911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076800.159917:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612643456:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076800.159918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076800.159923:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076800.159930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3788 00000100:00080000:3.0:1769076800.159935:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076800.159940:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076800.159946:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076800.159959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612643712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076800.159966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076800.159970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076800.159971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076800.159974:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3788 00000100:00080000:3.0:1769076800.159977:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076800.159981:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076800.159984:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076800.159997:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076800.160006:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612643840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076800.160016:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff8b10037dd180 x1855007612643456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076800.160032:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612643456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076800.160044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076800.160049:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076800.160053:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612643456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076800.160077:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643584:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076800.160103:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076800.160113:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff8b10037de300 x1855007612643584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076800.160124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612643584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076800.160133:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076800.160137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076800.160140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643584:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076800.160147:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076800.160169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076800.160181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff8b10037ddf80 x1855007612643712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076800.160192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612643712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076800.160197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076800.160200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076800.160202:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076800.160206:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076800.160220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076800.160227:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff8b10037dd500 x1855007612643840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076800.160234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612643840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076800.160254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076800.160257:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076800.160259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612643840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076801.055758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076801.055807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076802.079855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076802.079950:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076803.103798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076803.103887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076804.127738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076804.127801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076805.151743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076805.151795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076805.279768:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.279777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076805.279781:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.279789:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3793 00000100:00080000:2.0:1769076805.279800:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.279805:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076805.279837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612643968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.279861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.279864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076805.279865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.279868:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3793 00000100:00080000:2.0:1769076805.279870:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.279873:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076805.279880:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076805.279888:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612644096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.279902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.279905:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076805.279906:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.279909:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3793 00000100:00080000:2.0:1769076805.279912:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076805.279915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612643968:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076805.279916:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076805.279923:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076805.279931:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612644224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.279939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.279942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076805.279943:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.279945:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3793 00000100:00080000:2.0:1769076805.279948:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.279950:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076805.279953:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076805.279960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612644352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076805.279979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076805.280010:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff8b1005d59880 x1855007612643968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076805.280023:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612643968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076805.280031:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076805.280034:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076805.280037:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612643968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076805.280042:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644096:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076805.280055:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076805.280063:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff8b1005d59500 x1855007612644096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076805.280072:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612644096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076805.280078:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076805.280080:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076805.280082:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644096:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076805.280100:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076805.280111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076805.280117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff8b1005d5ad80 x1855007612644224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076805.280125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612644224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076805.280130:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076805.280132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076805.280134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076805.280138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076805.280148:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076805.280156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff8b1005d58380 x1855007612644352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076805.280168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612644352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076805.280176:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076805.280179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076805.280183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076806.175779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076806.175882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076807.199715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076807.199776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076808.223749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076808.223800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076809.247779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076809.247868:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076810.271739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076810.271814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076810.399804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076810.399820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076810.399827:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076810.399840:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3798 00000100:00080000:3.0:1769076810.399858:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076810.399879:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076810.399937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612644480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076810.399988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076810.399995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076810.399998:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076810.400006:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3798 00000100:00080000:3.0:1769076810.400014:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076810.400022:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076810.400043:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076810.400069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612644608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076810.400088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076810.400093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076810.400096:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076810.400102:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3798 00000100:00080000:3.0:1769076810.400109:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076810.400117:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612644480:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076810.400119:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076810.400135:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076810.400153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612644736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076810.400174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076810.400181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076810.400184:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076810.400208:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3798 00000100:00080000:3.0:1769076810.400219:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076810.400228:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076810.400238:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076810.400248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076810.400268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612644864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076810.400312:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff8b10037dc000 x1855007612644480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076810.400340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612644480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076810.400365:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076810.400371:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076810.400376:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612644480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076810.400386:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644608:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076810.400414:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076810.400429:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff8b10037dea00 x1855007612644608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076810.400444:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612644608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076810.400454:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076810.400458:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076810.400463:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076810.400495:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076810.400516:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076810.400528:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff8b10037dd880 x1855007612644736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076810.400542:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612644736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076810.400563:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076810.400605:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076810.400611:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076810.400620:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076810.400734:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076810.400758:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff8b10037df100 x1855007612644864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076810.400784:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612644864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076810.400800:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076810.400806:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076810.400813:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612644864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076811.295773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076811.295934:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076812.319751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076812.319835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076813.343722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076813.343780:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076814.367790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076814.367827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076815.391772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076815.391871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076815.519755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.519764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076815.519768:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.519776:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3803 00000100:00080000:3.0:1769076815.519786:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.519792:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076815.519864:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612644992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.519886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.519889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076815.519890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.519893:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3803 00000100:00080000:3.0:1769076815.519896:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.519899:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076815.519906:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076815.519915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612645120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.519923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.519925:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076815.519926:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.519929:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3803 00000100:00080000:3.0:1769076815.519931:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076815.519935:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612644992:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076815.519947:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076815.519951:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076815.519961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612645248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.519968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.519971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076815.519972:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.519975:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3803 00000100:00080000:3.0:1769076815.519977:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.519979:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076815.519982:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076815.519989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612645376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076815.520011:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076815.520043:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff8b10037ddc00 x1855007612644992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076815.520057:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612644992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076815.520065:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076815.520068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076815.520071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612644992:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076815.520077:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645120:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076815.520092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076815.520100:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff8b10037dce00 x1855007612645120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076815.520108:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612645120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076815.520114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076815.520116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076815.520118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076815.520135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076815.520149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076815.520155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff8b10037df480 x1855007612645248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076815.520163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612645248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076815.520168:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076815.520170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076815.520172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076815.520175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076815.520185:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076815.520192:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff8b10037df800 x1855007612645376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076815.520199:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612645376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076815.520204:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076815.520206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076815.520208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076816.415744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076816.415806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076817.439763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076817.439843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076818.463773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076818.463891:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076819.487734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076819.487843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076820.511823:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076820.511894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076820.639724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076820.639733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076820.639737:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076820.639745:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3808 00000100:00080000:2.0:1769076820.639753:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076820.639792:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076820.639821:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612645504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076820.639846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076820.639849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076820.639851:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076820.639854:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3808 00000100:00080000:2.0:1769076820.639857:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076820.639860:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076820.639869:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076820.639881:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612645632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076820.639893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076820.639897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076820.639898:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076820.639901:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3808 00000100:00080000:2.0:1769076820.639905:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076820.639908:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076820.639913:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076820.639922:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612645760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076820.639935:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612645504:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076820.639939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076820.639950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076820.639955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076820.639960:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3808 00000100:00080000:2.0:1769076820.639963:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076820.639965:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076820.639969:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076820.639980:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612645888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076820.640040:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076820.640109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff8b1005d5b100 x1855007612645504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076820.640141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612645504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076820.640160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076820.640165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076820.640169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612645504:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076820.640177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645632:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076820.640225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076820.640244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff8b1005d5aa00 x1855007612645632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076820.640262:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612645632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076820.640271:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076820.640276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076820.640281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645632:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076820.640321:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076820.640359:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076820.640369:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff8b1005d58380 x1855007612645760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076820.640378:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612645760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076820.640385:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076820.640387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076820.640389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076820.640393:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076820.640425:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076820.640433:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff8b1005d5ad80 x1855007612645888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076820.640467:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612645888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076820.640474:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076820.640476:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076820.640479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612645888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076821.535742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076821.535799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076822.559732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076822.559835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076823.583709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076823.583818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076824.607797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076824.608002:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076825.631718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076825.631829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076825.759815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076825.759826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076825.759831:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076825.759841:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3813 00000100:00080000:3.0:1769076825.759852:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076825.759859:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076825.759897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612646016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076825.759926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076825.759930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076825.759931:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076825.759936:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3813 00000100:00080000:3.0:1769076825.759940:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076825.759945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076825.759957:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076825.759977:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612646144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076825.759987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612646016:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076825.760010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076825.760016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076825.760019:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076825.760024:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3813 00000100:00080000:3.0:1769076825.760030:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076825.760034:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076825.760040:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076825.760059:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612646272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076825.760060:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076825.760079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076825.760083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076825.760085:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076825.760089:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3813 00000100:00080000:3.0:1769076825.760093:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076825.760097:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076825.760102:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076825.760108:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff8b10037dc700 x1855007612646016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076825.760116:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612646400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076825.760140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612646016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076825.760151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076825.760157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076825.760163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612646016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076825.760184:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646144:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076825.760205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076825.760216:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff8b10037df800 x1855007612646144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076825.760238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612646144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076825.760246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076825.760249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076825.760252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646144:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076825.760257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076825.760271:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076825.760279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff8b10037df480 x1855007612646272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076825.760290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612646272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076825.760297:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076825.760299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076825.760302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076825.760306:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076825.760362:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076825.760374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff8b10037dce00 x1855007612646400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076825.760385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612646400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076825.760394:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076825.760397:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076825.760400:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076826.655787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076826.655920:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076827.679799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076827.679958:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076828.703764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076828.703964:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076829.727736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076829.727918:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076830.751831:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076830.751905:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076830.879726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.879737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076830.879741:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.879751:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3818 00000100:00080000:3.0:1769076830.879762:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.879768:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076830.879804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612646528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.879831:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.879835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076830.879837:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.879841:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3818 00000100:00080000:3.0:1769076830.879844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.879847:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076830.879856:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076830.879865:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612646656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.879874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.879876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076830.879878:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.879880:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3818 00000100:00080000:3.0:1769076830.879883:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.879885:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076830.879889:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076830.879897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612646784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.879915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.879918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076830.879920:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.879924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3818 00000100:00080000:3.0:1769076830.879927:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.879929:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076830.879934:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076830.879940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612646528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076830.879947:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612646912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076830.880065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646656:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076830.880101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076830.880134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076831.775786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076831.775870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076832.799820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076832.799932:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076833.823796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076833.823941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076833.952012:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076833.952077:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076833.952108:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076833.952225:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076833.952251:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076833.952268:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769076833.952302:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076833] req@ffff8b10037dea00 x1855007612646912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076833.952351:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612646912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076833.952375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076833.952385:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076833.952393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076833.952409:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076833] req@ffff8b10037ddc00 x1855007612646528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076833.952432:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612646528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076833.952446:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076833.952453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076833.952461:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612646528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076833.952474:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076833] req@ffff8b10037df100 x1855007612646656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076833.952497:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612646656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076833.952512:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076833.952519:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076833.952525:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646656:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076833.952537:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076833] req@ffff8b10037dd880 x1855007612646784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076833.952562:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612646784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076833.952577:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076833.952583:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076833.952590:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612646784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076834.847727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076834.847759:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076835.871855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076835.872006:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076835.999810:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076835.999821:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076835.999825:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076835.999833:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3824 00000100:00080000:2.0:1769076835.999843:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076835.999849:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076835.999882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612647040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076835.999917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076835.999920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076835.999922:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076835.999925:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3824 00000100:00080000:2.0:1769076835.999928:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076835.999931:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076835.999939:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076835.999949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612647168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076835.999962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076835.999967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076835.999971:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076835.999978:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3824 00000100:00080000:2.0:1769076835.999983:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076835.999988:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076835.999996:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076836.000035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612647040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076836.000037:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612647296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076836.000073:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076836.000079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076836.000084:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076836.000092:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3824 00000100:00080000:2.0:1769076836.000099:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076836.000106:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076836.000115:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076836.000137:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612647424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076836.000209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076836.000296:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076836] req@ffff8b1005d59500 x1855007612647040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076836.000380:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612647040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076836.000405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076836.000414:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076836.000423:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612647040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076836.000440:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647168:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076836.000508:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076836.000538:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076836] req@ffff8b1005d59880 x1855007612647168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076836.000565:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612647168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076836.000592:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076836.000600:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076836.000606:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076836.000728:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076836.000779:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076836.000806:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff8b1005d5a300 x1855007612647296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076836.000834:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612647296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076836.000853:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076836.000860:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076836.000866:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076836.000881:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076836.000923:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076836.000949:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076836/real 1769076836] req@ffff8b1005d5bb80 x1855007612647424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076836.000975:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612647424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076836.000993:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076836.001001:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076836.001008:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076836.895778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076836.895866:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076837.919766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076837.919876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076838.943778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076838.943927:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076839.967719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076839.967873:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076840.991747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076840.991836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076841.119839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.119849:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076841.119853:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.119862:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3829 00000100:00080000:0.0:1769076841.119872:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.119878:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076841.119937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612647552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.119973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.119977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.119981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.119985:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3829 00000100:00080000:0.0:1769076841.119989:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.119993:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076841.120003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076841.120017:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612647680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.120029:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.120033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076841.120037:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076841.120042:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612647552:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076841.120045:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3829 00000100:00080000:0.0:1769076841.120103:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.120108:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076841.120113:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076841.120126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612647808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076841.120133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076841.120136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076841.120137:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076841.120140:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3829 00000100:00080000:0.0:1769076841.120174:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076841.120178:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076841.120185:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076841.120190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076841.120204:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612647936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076841.120243:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff8b1006024e00 x1855007612647552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076841.120266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612647552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076841.120279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076841.120284:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076841.120288:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612647552:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076841.120341:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647680:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769076841.120367:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076841.120381:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff8b1006024380 x1855007612647680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076841.120394:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612647680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076841.120413:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076841.120417:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076841.120421:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769076841.120427:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076841.120444:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076841.120469:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff8b1006027480 x1855007612647808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076841.120478:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612647808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076841.120484:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076841.120486:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076841.120488:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769076841.120500:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769076841.120531:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076841.120542:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff8b1006024700 x1855007612647936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076841.120553:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612647936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076841.120561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076841.120564:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769076841.120566:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612647936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076842.015731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076842.015790:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076843.039732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076843.039855:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076844.063852:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076844.064042:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076845.087713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076845.087852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076846.111742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076846.111804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076846.239777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076846.239788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076846.239792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076846.239802:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3834 00000100:00080000:3.0:1769076846.239813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076846.239820:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076846.239857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612648064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076846.239892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076846.239898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076846.239901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076846.239909:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3834 00000100:00080000:3.0:1769076846.239916:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076846.239922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076846.239931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612648064:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769076846.239933:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076846.239955:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612648192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076846.239981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076846.239986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076846.239988:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076846.239992:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3834 00000100:00080000:3.0:1769076846.239996:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076846.239999:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076846.240004:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076846.240015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612648320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076846.240027:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076846.240027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076846.240035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076846.240038:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076846.240045:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3834 00000100:00080000:3.0:1769076846.240051:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076846.240056:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076846.240063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076846.240067:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076846] req@ffff8b10037dc000 x1855007612648064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076846.240096:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612648064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076846.240100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612648448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076846.240108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076846.240116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076846.240122:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612648064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076846.240146:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648192:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076846.240167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076846.240181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076846] req@ffff8b10037dd500 x1855007612648192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076846.240192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612648192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076846.240200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076846.240203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076846.240206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648192:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076846.240211:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076846.240225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076846.240232:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076846] req@ffff8b10037ddf80 x1855007612648320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076846.240242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612648320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076846.240248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076846.240250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076846.240252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076846.240257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076846.240269:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076846.240276:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076846] req@ffff8b10037de300 x1855007612648448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076846.240286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612648448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076846.240292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076846.240294:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076846.240296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076847.135702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076847.135803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076848.159708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076848.159828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076849.183785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076849.183881:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076850.207722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076850.207799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076851.231730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076851.231805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076851.359844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076851.359858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076851.359864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076851.359876:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3839 00000100:00080000:2.0:1769076851.359945:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076851.359951:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076851.359984:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612648576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076851.360010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076851.360013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076851.360015:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076851.360018:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3839 00000100:00080000:2.0:1769076851.360020:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076851.360023:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076851.360030:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076851.360039:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612648704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076851.360048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076851.360053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076851.360056:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076851.360058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612648576:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076851.360062:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3839 00000100:00080000:2.0:1769076851.360068:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076851.360070:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076851.360074:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076851.360083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612648832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076851.360090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076851.360093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076851.360094:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076851.360097:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3839 00000100:00080000:2.0:1769076851.360099:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076851.360102:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076851.360104:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076851.360111:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612648960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076851.360121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076851.360153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff8b1005d5b480 x1855007612648576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076851.360167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612648576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076851.360175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076851.360180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076851.360185:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612648576:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076851.360209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648704:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076851.360235:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076851.360248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff8b1005d58e00 x1855007612648704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076851.360258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612648704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076851.360264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076851.360276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076851.360278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648704:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076851.360283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076851.360300:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076851.360308:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff8b1005d58000 x1855007612648832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076851.360317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612648832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076851.360322:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076851.360325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076851.360327:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076851.360331:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076851.360341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076851.360348:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff8b1005d58a80 x1855007612648960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076851.360355:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612648960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076851.360361:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076851.360363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076851.360365:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612648960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076852.255761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076852.255809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076853.279729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076853.279886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076854.303700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076854.303766:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076855.327792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076855.327981:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076856.351771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076856.351942:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076856.479772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076856.479789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076856.479792:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076856.479824:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3844 00000100:00080000:2.0:1769076856.479834:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076856.479841:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076856.479876:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612649088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076856.479903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076856.479908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076856.479910:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076856.479914:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3844 00000100:00080000:2.0:1769076856.479918:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076856.479923:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076856.479933:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076856.479949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612649216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076856.479960:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612649088:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076856.479962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076856.479965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076856.479968:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076856.479972:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3844 00000100:00080000:2.0:1769076856.479977:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076856.479981:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076856.479986:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076856.479996:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612649344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076856.480006:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076856.480010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076856.480013:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076856.480017:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3844 00000100:00080000:2.0:1769076856.480021:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769076856.480021:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076856.480026:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076856.480031:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076856.480041:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612649472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076856.480122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff8b1005d5b800 x1855007612649088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076856.480136:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612649088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076856.480258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076856.480265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076856.480271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612649088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076856.480344:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649216:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076856.480378:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076856.480395:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff8b1005d59180 x1855007612649216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076856.480410:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612649216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076856.480418:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076856.480420:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076856.480423:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076856.480429:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076856.480446:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076856.480455:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff8b1005d58a80 x1855007612649344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076856.480464:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612649344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076856.480487:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076856.480489:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076856.480491:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076856.480494:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076856.480506:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076856.480512:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff8b1005d58000 x1855007612649472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076856.480519:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612649472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076856.480524:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076856.480526:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076856.480528:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076857.375778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076857.375895:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076858.399814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076858.400014:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076859.423818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076859.423926:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076860.447833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076860.447898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076861.471834:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076861.471959:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076861.599841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076861.599854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076861.599859:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076861.599870:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3849 00000100:00080000:2.0:1769076861.599885:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076861.599894:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076861.599942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612649600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076861.599973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076861.599978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076861.599980:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076861.599984:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3849 00000100:00080000:2.0:1769076861.599988:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076861.599993:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076861.600003:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076861.600025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612649728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076861.600039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076861.600046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076861.600048:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076861.600054:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3849 00000100:00080000:2.0:1769076861.600059:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076861.600063:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076861.600070:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076861.600081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612649600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076861.600092:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612649856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076861.600107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076861.600110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076861.600112:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076861.600118:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3849 00000100:00080000:2.0:1769076861.600122:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076861.600127:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076861.600136:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076861.600149:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612649984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076861.600222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076861.600305:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff8b1005d58e00 x1855007612649600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076861.600346:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612649600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076861.600368:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076861.600375:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076861.600381:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612649600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076861.600393:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649728:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076861.600429:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076861.600448:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff8b1005d5b480 x1855007612649728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076861.600465:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612649728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076861.600476:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076861.600481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076861.600486:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649728:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076861.600526:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076861.600551:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076861.600564:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff8b1005d5bb80 x1855007612649856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076861.600579:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612649856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076861.600590:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076861.600594:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076861.600598:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076861.600606:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076861.600626:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076861.600681:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff8b1005d5a300 x1855007612649984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076861.600699:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612649984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076861.600737:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076861.600743:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076861.600747:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612649984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076862.495830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076862.495997:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076863.519807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076863.519995:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076864.543718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076864.543799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076865.567774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076865.567854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076866.591818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076866.591964:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076866.719860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.719877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076866.719883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.719898:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3854 00000100:00080000:0.0:1769076866.719916:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.719927:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076866.719989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612650112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.720034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.720039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076866.720041:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.720046:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3854 00000100:00080000:0.0:1769076866.720050:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.720054:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076866.720068:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076866.720084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612650240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.720107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.720113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769076866.720113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612650112:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076866.720116:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.720136:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3854 00000100:00080000:0.0:1769076866.720142:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.720145:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076866.720153:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076866.720179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612650368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.720191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.720196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076866.720199:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.720220:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3854 00000100:00080000:0.0:1769076866.720230:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.720235:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:3.0:1769076866.720242:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769076866.720244:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076866.720268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612650496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076866.720314:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff8b1006024700 x1855007612650112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076866.720341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612650112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076866.720357:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076866.720363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076866.720368:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612650112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076866.720398:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650240:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076866.720424:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076866.720437:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff8b1006027480 x1855007612650240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076866.720451:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612650240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076866.720460:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076866.720464:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076866.720468:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650240:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076866.720475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076866.720506:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076866.720517:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff8b1006024380 x1855007612650368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076866.720530:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612650368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076866.720539:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076866.720542:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076866.720545:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076866.720551:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076866.720585:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076866.720597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff8b1006024e00 x1855007612650496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076866.720610:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612650496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076866.720619:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076866.720692:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076866.720697:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076867.615761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076867.615874:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076868.639809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076868.639955:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076869.663760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076869.663837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076870.687822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076870.687980:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076871.711809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076871.711965:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076871.839734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076871.839744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076871.839748:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076871.839757:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3859 00000100:00080000:0.0:1769076871.839768:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076871.839775:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076871.839811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612650624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076871.839838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076871.839842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076871.839843:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076871.839847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3859 00000100:00080000:0.0:1769076871.839852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076871.839857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076871.839868:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076871.839884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612650752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076871.839899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076871.839905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076871.839908:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076871.839914:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3859 00000100:00080000:0.0:1769076871.839919:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076871.839924:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076871.839931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076871.839944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612650880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076871.839956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076871.839962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076871.839964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076871.839967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612650624:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076871.839970:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3859 00000100:00080000:0.0:1769076871.839975:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076871.839991:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076871.840000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076871.840024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612651008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076871.840117:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076871.840180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff8b1006024a80 x1855007612650624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076871.840208:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612650624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076871.840225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076871.840232:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076871.840238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612650624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076871.840249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650752:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076871.840279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076871.840295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff8b1006025f80 x1855007612650752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076871.840312:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612650752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076871.840322:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076871.840326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076871.840330:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076871.840365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076871.840390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076871.840403:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff8b1006027b80 x1855007612650880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076871.840427:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612650880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076871.840438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076871.840441:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076871.840445:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612650880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076871.840453:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076871.840473:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076871.840485:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff8b1006026d80 x1855007612651008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076871.840500:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612651008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076871.840510:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076871.840514:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076871.840518:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076872.735798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076872.735923:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076873.759749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076873.759848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076874.783784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076874.783827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076875.807707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076875.807771:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076876.831775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076876.831897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076876.959725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.959734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076876.959738:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.959746:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3865 00000100:00080000:0.0:1769076876.959755:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.959761:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076876.959793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612651136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.959816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.959820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076876.959822:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.959825:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3865 00000100:00080000:0.0:1769076876.959827:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.959829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076876.959837:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076876.959845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612651264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.959852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.959854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076876.959856:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.959858:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3865 00000100:00080000:0.0:1769076876.959860:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.959862:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076876.959865:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076876.959872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612651392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.959878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.959881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076876.959882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.959885:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3865 00000100:00080000:0.0:1769076876.959887:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.959891:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076876.959893:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612651136:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076876.959896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076876.959906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612651520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076876.959954:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076876.959985:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff8b1006026300 x1855007612651136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076876.960000:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612651136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076876.960008:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076876.960012:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076876.960015:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612651136:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076876.960020:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651264:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076876.960034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076876.960042:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff8b1006024000 x1855007612651264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076876.960051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612651264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076876.960056:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076876.960059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076876.960061:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651264:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076876.960065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076876.960076:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076876.960082:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff8b1006025880 x1855007612651392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076876.960090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612651392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076876.960100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076876.960103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076876.960105:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076876.960124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076876.960135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076876.960141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff8b1006026a00 x1855007612651520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076876.960149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612651520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076876.960154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076876.960156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076876.960158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076877.855770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076877.855921:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076878.879761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076878.879894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076879.903826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076879.903904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076880.927753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076880.927867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076881.951768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076881.951916:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076882.079785:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076882.079798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076882.079804:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076882.079827:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3870 00000100:00080000:0.0:1769076882.079842:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076882.079851:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076882.079911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612651648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076882.079967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076882.079972:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076882.079975:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076882.079980:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3870 00000100:00080000:0.0:1769076882.079985:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076882.079989:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076882.080000:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076882.080026:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612651776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076882.080048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076882.080052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076882.080054:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076882.080058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3870 00000100:00080000:0.0:1769076882.080062:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076882.080066:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076882.080072:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076882.080085:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612651904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076882.080107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076882.080112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076882.080115:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076882.080120:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3870 00000100:00080000:0.0:1769076882.080124:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076882.080131:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076882.080138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612651648:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076882.080140:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076882.080171:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612652032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076882.080247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076882.080308:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff8b1006025180 x1855007612651648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076882.080333:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612651648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.080348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076882.080354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076882.080360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612651648:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076882.080397:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651776:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076882.080426:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076882.080443:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff8b1006027800 x1855007612651776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076882.080460:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612651776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.080471:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076882.080475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076882.080479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076882.080487:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076882.080507:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076882.080531:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff8b1006027100 x1855007612651904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076882.080546:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612651904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.080556:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076882.080560:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076882.080563:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612651904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076882.080596:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076882.080617:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076882.080630:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff8b1006026680 x1855007612652032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076882.080692:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612652032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076882.080703:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076882.080707:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076882.080711:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076882.975781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076882.975940:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076883.999716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076883.999831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076885.023714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076885.023832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076886.047765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076886.047879:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076887.071777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076887.071845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076887.199782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.199792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076887.199796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.199804:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3875 00000100:00080000:1.0:1769076887.199815:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.199820:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076887.199855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612652160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.199894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.199909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076887.199910:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.199914:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3875 00000100:00080000:1.0:1769076887.199917:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.199920:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076887.199928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076887.199942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612652288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.199953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.199956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076887.199958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.199962:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3875 00000100:00080000:1.0:1769076887.199965:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.199973:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076887.199977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612652160:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769076887.199979:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076887.199996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612652416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.200005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.200008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076887.200010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.200017:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3875 00000100:00080000:1.0:1769076887.200022:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.200024:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076887.200028:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076887.200036:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612652544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076887.200090:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076887.200142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff8b1006137100 x1855007612652160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076887.200163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612652160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076887.200175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076887.200180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076887.200203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612652160:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076887.200241:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652288:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076887.200272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076887.200327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff8b1006136680 x1855007612652288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076887.200338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612652288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076887.200346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076887.200349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076887.200352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652288:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076887.200357:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076887.200398:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076887.200412:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff8b1006136d80 x1855007612652416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076887.200427:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612652416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076887.200438:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076887.200441:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076887.200446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076887.200453:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076887.200475:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076887.200487:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff8b1006135500 x1855007612652544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076887.200518:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612652544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076887.200525:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076887.200527:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076887.200530:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076888.095816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076888.095936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076889.119726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076889.119805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076890.143774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076890.143803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076891.167767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076891.167924:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076892.191830:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076892.191956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076892.319756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076892.319776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076892.319785:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076892.319805:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3880 00000100:00080000:1.0:1769076892.319826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076892.319839:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076892.319919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612652672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076892.319957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076892.319962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076892.319964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076892.319987:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3880 00000100:00080000:1.0:1769076892.319994:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076892.319999:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076892.320013:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076892.320034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612652800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076892.320046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076892.320053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076892.320055:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076892.320062:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3880 00000100:00080000:1.0:1769076892.320068:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076892.320075:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076892.320080:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076892.320091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612652672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076892.320094:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612652928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076892.320132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076892.320136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076892.320139:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076892.320143:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3880 00000100:00080000:1.0:1769076892.320147:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076892.320151:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076892.320157:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076892.320170:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612653056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076892.320265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076892.320357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff8b1006135880 x1855007612652672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076892.320399:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612652672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076892.320420:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076892.320428:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076892.320436:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612652672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076892.320453:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652800:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076892.320624:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076892.321003:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff8b1006137b80 x1855007612652800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076892.321124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612652800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076892.321145:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076892.321155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076892.321166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652800:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076892.321232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076892.321270:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076892.321291:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff8b1006135c00 x1855007612652928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076892.321311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612652928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076892.321324:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076892.321346:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076892.321352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612652928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076892.321362:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076892.321394:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076892.321415:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff8b1006136300 x1855007612653056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076892.321436:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612653056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076892.321460:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076892.321466:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076892.321472:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076893.215805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076893.215990:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076894.239802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076894.239958:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076895.263734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076895.263805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076896.287802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076896.287869:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076897.311718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076897.311918:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076897.439847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076897.439866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076897.439876:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076897.439896:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3885 00000100:00080000:1.0:1769076897.439918:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076897.439932:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076897.440006:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612653184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076897.440070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076897.440078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076897.440083:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076897.440095:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3885 00000100:00080000:1.0:1769076897.440104:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076897.440160:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612653184:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076897.440165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076897.440190:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076897.440221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612653312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076897.440246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076897.440253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076897.440257:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076897.440266:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3885 00000100:00080000:1.0:1769076897.440277:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076897.440286:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076897.440297:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653312:192.168.204.102@tcp:38:kworker.0 00000008:00100000:1.0:1769076897.440298:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076897.440339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612653440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076897.440367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076897.440374:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076897.440378:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076897.440386:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653440:192.168.204.102@tcp:8:kworker.0 00000100:00080000:1.0:1769076897.440389:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3885 00000100:00080000:1.0:1769076897.440402:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076897.440412:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076897.440424:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076897.440489:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612653568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076897.440568:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076898.335787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076898.335929:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076899.359756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076899.359893:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076900.383826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076900.384000:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076900.512222:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769076900.512296:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076900.512322:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076900.512378:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076900.512405:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769076900.512421:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769076900.512484:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076900] req@ffff8b1006136300 x1855007612653568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076900.512543:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612653568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076900.512579:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076900.512589:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076900.512596:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076900.512612:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076900] req@ffff8b1006136a00 x1855007612653440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076900.512733:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612653440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076900.512748:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076900.512754:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076900.512760:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076900.512778:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076900] req@ffff8b1006135180 x1855007612653312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076900.512797:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612653312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076900.512810:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076900.512815:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076900.512820:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653312:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076900.512830:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076900] req@ffff8b1006137800 x1855007612653184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076900.512848:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612653184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076900.512860:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076900.512865:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076900.512870:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612653184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076901.407706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076901.407897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076902.431793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076902.431931:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076902.559859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.559870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076902.559874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.559883:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3890 00000100:00080000:2.0:1769076902.559894:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.559902:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076902.559970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612653696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.560041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.560049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076902.560053:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.560062:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3890 00000100:00080000:2.0:1769076902.560083:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.560088:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076902.560098:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076902.560110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612653696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076902.560115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612653824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.560125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.560128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076902.560146:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.560149:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3890 00000100:00080000:2.0:1769076902.560153:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.560158:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076902.560171:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076902.560185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612653952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076902.560196:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076902.560208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.560212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076902.560216:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.560222:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3890 00000100:00080000:2.0:1769076902.560228:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076902.560233:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff8b1005d59880 x1855007612653696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076902.560235:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076902.560245:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076902.560258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612653696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076902.560264:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612654080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076902.560300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076902.560307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076902.560312:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612653696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076902.560349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653824:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076902.560377:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076902.560391:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff8b1005d59500 x1855007612653824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076902.560405:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612653824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076902.560414:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076902.560418:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076902.560422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653824:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076902.560429:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076902.560455:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076902.560468:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff8b1005d5ad80 x1855007612653952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076902.560492:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612653952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076902.560500:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076902.560504:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076902.560508:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612653952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076902.560515:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076902.560557:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076902.560581:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff8b1005d58380 x1855007612654080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076902.560596:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612654080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076902.560610:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076902.560614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076902.560618:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076903.455736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076903.456042:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076904.479746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076904.479852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076905.503783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076905.503900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076906.527764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076906.527944:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076907.551809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076907.551910:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076907.679904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.679917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076907.679923:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.679957:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3895 00000100:00080000:0.0:1769076907.679989:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.679997:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076907.680062:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612654208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.680105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.680131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076907.680135:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.680142:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3895 00000100:00080000:0.0:1769076907.680148:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.680157:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076907.680171:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076907.680212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612654336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.680233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.680239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769076907.680242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612654208:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076907.680261:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.680283:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3895 00000100:00080000:0.0:1769076907.680290:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.680296:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076907.680305:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076907.680327:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612654464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.680339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.680344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076907.680347:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.680351:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3895 00000100:00080000:0.0:1769076907.680356:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.680360:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076907.680367:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076907.680377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612654592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076907.680407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076907.680492:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff8b1006026680 x1855007612654208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076907.680523:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612654208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076907.680539:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076907.680546:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076907.680552:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612654208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076907.680592:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654336:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076907.680750:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076907.680775:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff8b1006027100 x1855007612654336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076907.680802:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612654336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076907.680820:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076907.680828:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076907.680835:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654336:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076907.680850:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076907.680897:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076907.680916:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff8b1006027800 x1855007612654464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076907.680973:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612654464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076907.680985:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076907.680989:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076907.680994:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076907.681003:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076907.681034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076907.681049:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff8b1006025180 x1855007612654592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076907.681065:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612654592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076907.681075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076907.681079:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076907.681083:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076908.575738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076908.575887:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076909.599768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076909.599854:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076910.623787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076910.623935:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076911.647769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076911.647925:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076912.671747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076912.671793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076912.799815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076912.799845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076912.799890:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076912.799904:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3900 00000100:00080000:0.0:1769076912.799919:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076912.799928:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076912.800000:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612654720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076912.800038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076912.800043:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076912.800046:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076912.800050:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3900 00000100:00080000:0.0:1769076912.800054:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076912.800058:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076912.800072:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076912.800087:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612654848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076912.800106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612654720:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076912.800114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076912.800120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076912.800123:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076912.800128:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3900 00000100:00080000:0.0:1769076912.800132:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076912.800136:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076912.800143:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076912.800168:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612654976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076912.800178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076912.800182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076912.800184:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076912.800189:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3900 00000100:00080000:0.0:1769076912.800194:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076912.800198:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076912.800203:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076912.800214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076912.800220:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612655104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076912.800294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076912] req@ffff8b1006026a00 x1855007612654720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076912.800326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612654720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076912.800338:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076912.800359:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076912.800364:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612654720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076912.800396:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654848:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076912.800428:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076912.800443:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076912] req@ffff8b1006025880 x1855007612654848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076912.800458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612654848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076912.800468:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076912.800472:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076912.800476:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076912.800482:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076912.800503:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076912.800513:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076912] req@ffff8b1006024000 x1855007612654976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076912.800526:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612654976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076912.800551:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076912.800572:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076912.800577:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612654976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076912.800586:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076912.800617:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076912.800635:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076912] req@ffff8b1006026300 x1855007612655104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076912.800668:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612655104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076912.800678:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076912.800682:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076912.800686:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076913.695719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076913.695782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076914.719742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076914.719861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076915.743728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076915.743790:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076916.767728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076916.767775:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076917.791760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076917.791834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076917.919823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076917.919835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076917.919840:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076917.919851:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3906 00000100:00080000:2.0:1769076917.919863:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076917.919870:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076917.919924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612655232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076917.919956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076917.919961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076917.919963:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076917.919968:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3906 00000100:00080000:2.0:1769076917.919973:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076917.919978:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076917.919989:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076917.920003:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612655232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076917.920008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612655360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076917.920024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076917.920028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076917.920031:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076917.920035:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3906 00000100:00080000:2.0:1769076917.920040:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076917.920045:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076917.920052:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076917.920078:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612655488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076917.920080:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076917.920095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076917.920100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076917.920103:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076917.920109:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3906 00000100:00080000:2.0:1769076917.920115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076917.920123:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076917.920123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff8b1005d5aa00 x1855007612655232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076917.920134:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076917.920146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612655232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076917.920153:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612655616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076917.920160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076917.920168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076917.920190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612655232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076917.920210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655360:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076917.920229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076917.920238:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff8b1005d5b100 x1855007612655360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076917.920246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612655360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076917.920252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076917.920255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076917.920257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655360:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076917.920261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076917.920298:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076917.920310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff8b1005d58380 x1855007612655488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076917.920325:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612655488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076917.920335:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076917.920339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076917.920342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076917.920350:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076917.920368:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076917.920377:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff8b1005d5ad80 x1855007612655616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076917.920386:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612655616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076917.920403:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076917.920406:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076917.920408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076918.815769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076918.815870:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076919.839797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076919.839955:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076920.863837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076920.863920:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076921.887793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076921.887951:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076922.911773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076922.911887:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076923.039778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.039786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076923.039789:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.039798:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3911 00000100:00080000:3.0:1769076923.039807:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.039812:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076923.039844:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612655744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.039869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.039872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.039874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.039877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3911 00000100:00080000:3.0:1769076923.039880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.039882:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076923.039889:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076923.039898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612655872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.039905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.039908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076923.039909:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.039912:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3911 00000100:00080000:3.0:1769076923.039914:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.039916:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076923.039919:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076923.039927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612655744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076923.039927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612656000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076923.039941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076923.039943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076923.039945:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076923.039948:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3911 00000100:00080000:3.0:1769076923.039950:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076923.039952:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076923.039956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076923.039963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612656128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076923.039999:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076923.040037:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff8b10037dd180 x1855007612655744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.040057:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612655744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.040068:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.040073:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076923.040077:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612655744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076923.040086:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655872:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076923.040110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076923.040123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff8b10037ded80 x1855007612655872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.040136:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612655872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.040144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.040148:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076923.040151:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612655872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076923.040178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.040199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076923.040211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff8b10037de680 x1855007612656000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.040224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612656000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.040233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.040236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076923.040240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076923.040246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076923.040265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076923.040278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076923/real 1769076923] req@ffff8b10037dc380 x1855007612656128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076923.040290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612656128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076923.040298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076923.040302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076923.040305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076923.935763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076923.935811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076924.959753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076924.959820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076925.983724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076925.983806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076927.007785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076927.007853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076928.031782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076928.031834:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076928.159815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076928.159833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076928.159844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076928.159858:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3916 00000100:00080000:3.0:1769076928.159874:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076928.159883:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076928.159927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612656256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076928.159978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076928.159987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076928.159991:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076928.160000:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3916 00000100:00080000:3.0:1769076928.160007:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076928.160015:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076928.160033:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076928.160066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612656384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076928.160089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612656256:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076928.160095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076928.160104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076928.160107:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076928.160115:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3916 00000100:00080000:3.0:1769076928.160123:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076928.160130:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076928.160139:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076928.160159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612656512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076928.160190:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076928.160193:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076928.160195:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076928.160202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076928.160206:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3916 00000100:00080000:3.0:1769076928.160215:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076928.160229:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076928.160236:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076928.160253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff8b10037dca80 x1855007612656256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076928.160274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612656256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076928.160287:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076928.160293:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076928.160298:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612656256:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076928.160347:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076928.160381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612656640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076928.160396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076928.160428:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff8b10037dc380 x1855007612656384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076928.160453:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612656384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076928.160466:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076928.160472:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076928.160479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076928.160488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076928.160511:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076928.160525:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff8b10037de680 x1855007612656512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076928.160540:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612656512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076928.160561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076928.160566:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076928.160569:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076928.160709:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076928.160752:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076928.160770:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff8b10037ded80 x1855007612656640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076928.160785:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612656640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076928.160797:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076928.160804:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076928.160810:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076929.055829:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076929.055935:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076930.079860:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076930.079897:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076931.103725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076931.103794:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076932.127800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076932.127995:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076933.151762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076933.151872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076933.279737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076933.279745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076933.279748:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076933.279756:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3921 00000100:00080000:1.0:1769076933.279765:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076933.279771:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076933.279805:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612656768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076933.279827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076933.279830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076933.279832:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076933.279835:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3921 00000100:00080000:1.0:1769076933.279837:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076933.279840:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076933.279847:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076933.279856:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612656896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076933.279864:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612656768:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769076933.279866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076933.279874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076933.279875:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076933.279879:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3921 00000100:00080000:1.0:1769076933.279882:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076933.279884:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076933.279888:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076933.279902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612657024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076933.279908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076933.279911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076933.279912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076933.279916:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3921 00000100:00080000:1.0:1769076933.279918:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076933.279920:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076933.279923:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076933.279926:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769076933.279934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612657152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076933.279959:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff8b1006135c00 x1855007612656768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076933.279974:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612656768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076933.279982:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076933.279985:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076933.279988:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612656768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076933.280006:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656896:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076933.280021:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076933.280029:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff8b1006137b80 x1855007612656896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076933.280037:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612656896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076933.280043:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076933.280045:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076933.280047:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612656896:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076933.280051:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076933.280062:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076933.280068:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff8b1006135880 x1855007612657024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076933.280075:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612657024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076933.280081:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076933.280082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076933.280084:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076933.280088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076933.280109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076933.280115:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff8b1006135500 x1855007612657152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076933.280123:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612657152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076933.280128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076933.280139:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076933.280141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076934.175721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076934.175892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076935.199764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076935.199814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076936.223734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076936.223855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076937.247778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076937.247903:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076938.271715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076938.271807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076938.399727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076938.399738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076938.399742:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076938.399750:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3926 00000100:00080000:1.0:1769076938.399760:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076938.399766:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769076938.399800:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612657280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076938.399825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076938.399828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076938.399829:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076938.399833:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3926 00000100:00080000:1.0:1769076938.399835:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076938.399837:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076938.399845:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076938.399852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612657408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076938.399871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076938.399874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076938.399875:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076938.399877:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3926 00000100:00080000:1.0:1769076938.399879:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076938.399881:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076938.399884:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076938.399890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612657536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076938.399897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076938.399900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076938.399901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076938.399904:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3926 00000100:00080000:1.0:1769076938.399906:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076938.399909:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769076938.399913:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076938.399921:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612657280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769076938.399922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612657664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076938.399991:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076938.400021:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff8b1006136d80 x1855007612657280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076938.400036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612657280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076938.400044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076938.400047:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076938.400050:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612657280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076938.400056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657408:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076938.400069:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076938.400077:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff8b1006136680 x1855007612657408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076938.400085:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612657408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076938.400096:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076938.400098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076938.400100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076938.400104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076938.400115:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076938.400121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff8b1006137100 x1855007612657536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076938.400128:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612657536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076938.400134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076938.400136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076938.400138:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076938.400158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076938.400169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076938.400176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff8b1006134700 x1855007612657664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076938.400189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612657664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076938.400210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076938.400214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076938.400218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076939.295780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076939.295866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076940.319790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076940.319857:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076941.343745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076941.343827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076942.367790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076942.367894:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076943.391755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076943.391887:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076943.519917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.519931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076943.519936:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.519948:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3931 00000100:00080000:0.0:1769076943.519963:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.519984:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076943.520033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612657792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076943.520072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.520078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076943.520082:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.520088:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3931 00000100:00080000:0.0:1769076943.520092:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.520099:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076943.520110:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076943.520140:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612657920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076943.520155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.520159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076943.520162:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.520166:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3931 00000100:00080000:0.0:1769076943.520170:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.520180:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076943.520180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612657792:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769076943.520189:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076943.520212:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612658048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076943.520232:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.520239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076943.520244:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.520252:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3931 00000100:00080000:0.0:1769076943.520259:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.520275:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076943.520284:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076943.520305:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612658176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076943.520326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076943.520439:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff8b1006026d80 x1855007612657792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076943.520479:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612657792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076943.520493:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076943.520499:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076943.520504:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612657792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076943.520515:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657920:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076943.520562:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076943.520578:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff8b1006027b80 x1855007612657920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076943.520594:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612657920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076943.520616:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076943.520620:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076943.520625:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612657920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076943.520701:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076943.520729:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076943.520745:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff8b1006025f80 x1855007612658048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076943.520771:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612658048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076943.520782:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076943.520786:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076943.520790:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076943.520797:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076943.520818:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076943.520830:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff8b1006024a80 x1855007612658176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076943.520854:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612658176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076943.520864:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076943.520868:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076943.520871:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076944.415779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076944.415879:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076945.439725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076945.439797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076946.463737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076946.463806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076947.487737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076947.487871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076948.511776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076948.511895:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076948.639828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.639842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076948.639848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.639861:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3936 00000100:00080000:0.0:1769076948.639876:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.639886:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076948.639934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612658304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.639971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.639976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076948.639979:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.639984:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3936 00000100:00080000:0.0:1769076948.639989:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.639993:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076948.640004:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076948.640019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612658432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.640047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.640052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076948.640055:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.640061:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3936 00000100:00080000:0.0:1769076948.640067:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076948.640073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612658304:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076948.640078:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076948.640094:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076948.640123:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612658560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.640138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.640143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076948.640145:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.640149:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3936 00000100:00080000:0.0:1769076948.640153:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.640174:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076948.640179:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076948.640189:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612658688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076948.640214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076948.640268:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff8b1006024e00 x1855007612658304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076948.640312:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612658304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076948.640332:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076948.640341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076948.640349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612658304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076948.640364:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658432:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076948.640411:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076948.640438:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff8b1006024380 x1855007612658432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076948.640465:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612658432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076948.640476:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076948.640481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076948.640485:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076948.640516:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076948.640546:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076948.640560:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff8b1006027480 x1855007612658560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076948.640575:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612658560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076948.640586:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076948.640590:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076948.640594:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076948.640601:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076948.640622:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076948.640633:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff8b1006024700 x1855007612658688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076948.640648:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612658688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076948.640713:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076948.640717:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076948.640720:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076949.535767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076949.535867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076950.559761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076950.559914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076951.583752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076951.583945:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076952.607731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076952.607877:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076953.631772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076953.631856:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076953.759757:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076953.759771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076953.759778:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076953.759793:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3941 00000100:00080000:0.0:1769076953.759828:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076953.759840:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076953.759947:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612658816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076953.759995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076953.760000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076953.760003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076953.760009:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3941 00000100:00080000:0.0:1769076953.760033:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076953.760039:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076953.760057:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076953.760086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612658944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076953.760107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076953.760114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076953.760119:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076953.760127:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3941 00000100:00080000:0.0:1769076953.760137:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076953.760143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612658816:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076953.760150:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076953.760191:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076953.760234:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612659072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076953.760260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076953.760268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076953.760272:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076953.760282:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3941 00000800:00000100:2.0:1769076953.760293:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769076953.760294:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076953.760310:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076953.760325:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076953.760350:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff8b1006024700 x1855007612658816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076953.760359:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612659200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076953.760448:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612658816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076953.760463:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076953.760469:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076953.760476:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612658816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076953.760488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658944:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076953.760523:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076953.760536:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff8b1006027480 x1855007612658944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076953.760550:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612658944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076953.760558:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076953.760562:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076953.760565:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612658944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076953.760602:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076953.760619:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076953.760629:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff8b1006024380 x1855007612659072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076953.760698:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612659072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076953.760707:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076953.760711:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076953.760714:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076953.760721:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076953.760741:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076953.760765:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff8b1006024e00 x1855007612659200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076953.760777:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612659200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076953.760791:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076953.760794:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076953.760797:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076954.655763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076954.655948:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076955.679728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076955.679842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076956.703807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076956.703941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076957.727756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076957.727868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076958.751767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076958.751844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076958.879858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076958.879868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076958.879873:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076958.879885:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3946 00000100:00080000:0.0:1769076958.879898:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076958.879906:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076958.879947:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612659328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076958.879974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076958.879977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076958.879979:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076958.879983:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3946 00000100:00080000:0.0:1769076958.879986:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076958.879989:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076958.879998:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076958.880010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612659456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076958.880020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076958.880026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769076958.880026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612659328:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769076958.880030:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076958.880038:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3946 00000100:00080000:0.0:1769076958.880041:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076958.880043:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076958.880047:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076958.880057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612659584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076958.880064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076958.880067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076958.880069:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076958.880072:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3946 00000100:00080000:0.0:1769076958.880075:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076958.880133:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076958.880137:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076958.880147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612659712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076958.880160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076958.880194:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff8b1006024a80 x1855007612659328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.880210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612659328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.880218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.880222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076958.880225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612659328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076958.880243:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076958.880274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076958.880283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff8b1006025f80 x1855007612659456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.880293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612659456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.880299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.880301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076958.880304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076958.880309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076958.880333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076958.880340:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff8b1006027b80 x1855007612659584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.880348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612659584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.880354:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.880356:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076958.880358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076958.880362:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076958.880374:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076958.880380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff8b1006026d80 x1855007612659712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.880400:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612659712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.880406:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.880408:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076958.880410:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076959.775722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076959.775819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076960.799756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076960.800042:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076961.823855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076961.823974:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076962.847751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076962.847829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076963.871764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076963.871839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076963.999729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.999740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076963.999754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.999765:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3952 00000100:00080000:0.0:1769076963.999777:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.999796:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769076963.999830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612659840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.999862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.999866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076963.999868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.999872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3952 00000100:00080000:0.0:1769076963.999877:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.999881:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076963.999891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076963.999908:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612659840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076963.999909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612659968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.999927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.999931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076963.999933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.999938:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3952 00000100:00080000:0.0:1769076963.999943:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.999948:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076963.999954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076963.999969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612660096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.999981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.999985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076963.999987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076964.000007:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3952 00000100:00080000:0.0:1769076964.000023:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076964.000028:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769076964.000037:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076964.000051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612660224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076964.000095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076964.000137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076964.000154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076964.895706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076964.895773:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076965.919773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076965.919890:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076966.943776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076966.943924:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076967.072123:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769076967.072209:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076967.072239:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076967.072319:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076967.072339:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769076967.072353:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769076967.072628:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076967] req@ffff8b1006026a00 x1855007612660224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076967.072713:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612660224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076967.072735:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076967.072744:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076967.072752:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076967.072768:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076967] req@ffff8b1006024000 x1855007612659968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076967.072799:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612659968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076967.072816:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076967.072823:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076967.072830:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612659968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076967.072842:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076967] req@ffff8b1006025880 x1855007612660096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076967.072865:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612660096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076967.072881:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076967.072888:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076967.072894:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076967.072906:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076967] req@ffff8b1006026300 x1855007612659840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076967.072931:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612659840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076967.072977:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076967.072986:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076967.072994:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612659840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076967.967790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076967.967894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076968.991819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076968.991878:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076969.119748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076969.119757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076969.119761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076969.119770:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3957 00000100:00080000:2.0:1769076969.119779:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076969.119785:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076969.119814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612660352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076969.119837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076969.119840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076969.119841:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076969.119844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3957 00000100:00080000:2.0:1769076969.119846:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076969.119849:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076969.119855:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.119865:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612660480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076969.119872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076969.119876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076969.119878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076969.119881:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3957 00000100:00080000:2.0:1769076969.119883:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076969.119885:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076969.119888:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.119897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612660608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076969.119936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076969.119939:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612660352:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076969.119940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076969.119942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076969.119954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3957 00000100:00080000:2.0:1769076969.119962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076969.119964:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076969.119967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.119976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612660736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076969.120077:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076969.120136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff8b1005d59500 x1855007612660352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076969.120162:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612660352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076969.120177:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076969.120183:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076969.120188:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612660352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076969.120203:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076969.120236:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076969.120253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff8b1005d59880 x1855007612660480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076969.120270:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612660480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076969.120281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076969.120285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076969.120289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076969.120335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076969.120363:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076969.120379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff8b1005d5a300 x1855007612660608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076969.120395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612660608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076969.120406:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076969.120411:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076969.120415:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076969.120424:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076969.120451:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076969.120466:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff8b1005d5bb80 x1855007612660736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076969.120483:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612660736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076969.120493:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076969.120497:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076969.120502:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076970.015727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076970.015820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076971.039755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076971.039859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076972.063801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076972.063904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076973.087743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076973.087815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076974.111788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076974.111888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076974.239812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076974.239840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076974.239854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076974.239864:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3962 00000100:00080000:2.0:1769076974.239876:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076974.239884:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076974.239922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612660864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076974.239947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076974.239952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076974.239971:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076974.239978:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3962 00000100:00080000:2.0:1769076974.239984:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076974.239990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076974.239999:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612660864:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769076974.240005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076974.240029:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612660992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076974.240064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076974.240070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076974.240074:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076974.240081:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3962 00000100:00080000:2.0:1769076974.240089:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769076974.240097:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769076974.240104:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076974.240113:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076974.240132:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612661120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076974.240149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076974.240153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff8b1005d5b480 x1855007612660864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076974.240159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076974.240183:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076974.240195:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3962 00000100:00080000:2.0:1769076974.240204:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076974.240210:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076974.240217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612660864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076974.240220:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1769076974.240257:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076974.240259:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612661248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076974.240271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076974.240278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612660864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076974.240303:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076974.240327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076974.240339:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff8b1005d58e00 x1855007612660992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076974.240350:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612660992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076974.240357:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076974.240359:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076974.240362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612660992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076974.240367:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076974.240380:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076974.240388:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff8b1005d58000 x1855007612661120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076974.240397:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612661120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076974.240403:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076974.240406:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076974.240408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076974.240412:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076974.240433:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076974.240440:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff8b1005d58a80 x1855007612661248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076974.240449:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612661248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076974.240461:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076974.240463:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076974.240466:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076975.135773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076975.135794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076976.159746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076976.159890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076977.183727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076977.183905:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076978.207732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076978.207846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076979.231814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076979.231937:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076979.359806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.359816:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076979.359840:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.359851:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3967 00000100:00080000:2.0:1769076979.359863:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.359871:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076979.359911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612661376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.359958:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.359962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076979.359964:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.359968:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3967 00000100:00080000:2.0:1769076979.359972:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.359998:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076979.360006:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076979.360019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612661504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.360028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.360032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076979.360035:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076979.360038:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612661376:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076979.360040:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3967 00000100:00080000:2.0:1769076979.360047:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.360051:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076979.360055:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076979.360065:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612661632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.360086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.360088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076979.360090:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.360093:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3967 00000100:00080000:2.0:1769076979.360102:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.360106:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076979.360109:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076979.360125:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612661760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076979.360129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076979.360170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076979] req@ffff8b1005d59180 x1855007612661376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076979.360186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612661376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076979.360195:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076979.360198:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076979.360201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612661376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076979.360221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661504:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076979.360239:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076979.360249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076979] req@ffff8b1005d5b800 x1855007612661504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076979.360258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612661504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076979.360266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076979.360268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076979.360271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076979.360275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076979.360287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076979.360294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076979] req@ffff8b1005d58a80 x1855007612661632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076979.360302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612661632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076979.360309:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076979.360311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076979.360313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076979.360317:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076979.360328:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076979.360336:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076979] req@ffff8b1005d58000 x1855007612661760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076979.360351:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612661760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076979.360360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076979.360363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076979.360367:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612661760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076980.255714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076980.255797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769076981.279807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076981.279861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076982.303748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076982.303804:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076983.327751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076983.327808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076984.351718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076984.351782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076984.479811:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.479823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076984.479829:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.479840:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3972 00000100:00080000:2.0:1769076984.479852:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.479860:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769076984.479901:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612661888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.479937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.479943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076984.479945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.479950:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3972 00000100:00080000:2.0:1769076984.479955:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.479960:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076984.479972:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076984.480003:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612662016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.480021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.480026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076984.480030:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076984.480031:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612661888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769076984.480038:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3972 00000100:00080000:2.0:1769076984.480062:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.480077:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076984.480083:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076984.480099:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612662144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.480110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.480114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076984.480116:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.480120:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3972 00000100:00080000:2.0:1769076984.480125:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.480129:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769076984.480135:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076984.480143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076984.480149:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612662272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076984.480190:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff8b1005d58e00 x1855007612661888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076984.480211:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612661888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076984.480224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076984.480229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076984.480233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612661888:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076984.480260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662016:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769076984.480293:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076984.480308:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff8b1005d5b480 x1855007612662016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076984.480323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612662016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076984.480332:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076984.480335:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076984.480340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769076984.480347:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076984.480368:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076984.480380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff8b1005d5bb80 x1855007612662144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076984.480393:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612662144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076984.480402:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076984.480405:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076984.480409:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076984.480415:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769076984.480435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076984.480447:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff8b1005d5a300 x1855007612662272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076984.480460:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612662272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076984.480469:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076984.480472:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769076984.480476:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076985.375770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076985.375828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076986.399734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076986.399794:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076987.423751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076987.423808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076988.447771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076988.447879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076989.471723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076989.471775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076989.599824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076989.599852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076989.599859:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076989.599873:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3977 00000100:00080000:3.0:1769076989.599891:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076989.599902:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076989.599971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612662400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076989.600018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076989.600023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076989.600026:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076989.600032:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3977 00000100:00080000:3.0:1769076989.600036:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076989.600041:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076989.600054:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076989.600070:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612662528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076989.600086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076989.600091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076989.600094:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076989.600098:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3977 00000100:00080000:3.0:1769076989.600112:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076989.600119:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076989.600125:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076989.600135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612662400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769076989.600147:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612662656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076989.600164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076989.600169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076989.600172:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076989.600178:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3977 00000100:00080000:3.0:1769076989.600182:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076989.600187:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076989.600194:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076989.600207:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612662784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076989.600225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076989.600270:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff8b10037dd180 x1855007612662400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.600288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612662400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.600315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.600319:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076989.600322:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612662400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076989.600331:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662528:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076989.600349:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076989.600357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff8b10037de300 x1855007612662528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.600367:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612662528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.600388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.600391:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076989.600393:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076989.600413:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076989.600427:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076989.600434:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff8b10037ddf80 x1855007612662656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.600460:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612662656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.600472:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.600475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076989.600478:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076989.600483:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076989.600497:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076989.600521:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff8b10037dd500 x1855007612662784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.600531:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612662784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.600539:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.600541:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076989.600544:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612662784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076990.495811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076990.495944:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076991.519799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076991.519985:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076992.543744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076992.543835:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076993.567740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076993.567869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076994.591791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076994.591867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076994.719759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076994.719772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076994.719784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076994.719797:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3982 00000100:00080000:3.0:1769076994.719811:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076994.719820:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076994.719869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612662912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076994.719908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076994.719915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076994.719918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076994.719926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3982 00000100:00080000:3.0:1769076994.719932:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076994.719940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076994.719957:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076994.719981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612663040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076994.719990:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612662912:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076994.720003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076994.720009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076994.720013:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076994.720019:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3982 00000100:00080000:3.0:1769076994.720026:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076994.720032:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076994.720041:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076994.720057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612663168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076994.720073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076994.720080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769076994.720081:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769076994.720084:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076994.720090:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3982 00000100:00080000:3.0:1769076994.720096:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076994.720103:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076994.720112:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076994.720130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612663296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076994.720154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff8b10037dc000 x1855007612662912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.720188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612662912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.720209:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.720217:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076994.720225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612662912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769076994.720265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769076994.720303:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076994.720322:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff8b10037dd880 x1855007612663040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.720338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612663040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.720349:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.720353:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076994.720358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769076994.720365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076994.720390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076994.720402:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff8b10037df100 x1855007612663168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.720417:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612663168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.720427:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.720431:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076994.720434:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769076994.720441:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769076994.720461:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769076994.720535:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff8b10037ddc00 x1855007612663296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.720564:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612663296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.720583:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.720589:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769076994.720594:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769076995.615897:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076995.615911:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076996.639766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076996.639934:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076997.663803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076997.663875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076998.687805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076998.687858:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076999.711750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076999.711928:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076999.839748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076999.839760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076999.839765:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076999.839778:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3987 00000100:00080000:3.0:1769076999.839792:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076999.839801:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769076999.839846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612663424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076999.839894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076999.839899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076999.839902:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076999.839908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3987 00000100:00080000:3.0:1769076999.839915:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076999.839922:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076999.839952:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076999.839978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612663552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076999.840001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076999.840004:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612663424:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769076999.840008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076999.840028:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076999.840038:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3987 00000100:00080000:3.0:1769076999.840045:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076999.840054:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076999.840063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076999.840084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612663680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076999.840101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076999.840106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076999.840109:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076999.840115:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3987 00000100:00080000:3.0:1769076999.840122:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076999.840128:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769076999.840136:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076999.840151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769076999.840163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612663808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076999.840266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff8b10037dea00 x1855007612663424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076999.840316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612663424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076999.840333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076999.840340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076999.840346:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612663424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769076999.840467:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769076999.840516:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076999.840539:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff8b10037dce00 x1855007612663552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076999.840573:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612663552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076999.840585:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076999.840591:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076999.840595:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769076999.840605:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076999.840636:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076999.840649:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff8b10037df480 x1855007612663680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076999.840728:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612663680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076999.840740:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076999.840746:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076999.840751:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769076999.840760:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769076999.840805:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769076999.840856:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff8b10037df800 x1855007612663808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076999.840889:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612663808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076999.840908:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076999.840915:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769076999.840919:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612663808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077000.735786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077000.735919:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077001.759851:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077001.759929:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077002.783786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077002.783907:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077003.807719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077003.807870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077004.831791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077004.831899:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077004.959807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.959826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077004.959845:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.959858:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3993 00000100:00080000:3.0:1769077004.959873:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.959882:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077004.959939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612663936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.959998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.960004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077004.960008:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.960014:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3993 00000100:00080000:3.0:1769077004.960030:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.960054:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077004.960069:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077004.960106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612664064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.960127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.960133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077004.960137:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077004.960145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612663936:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077004.960151:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3993 00000100:00080000:3.0:1769077004.960164:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.960172:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077004.960184:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077004.960209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612664192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.960231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.960238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077004.960245:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077004.960248:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.960263:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3993 00000100:00080000:3.0:1769077004.960273:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.960288:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077004.960346:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077004.960358:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff8b10037dc700 x1855007612663936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077004.960369:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612664320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077004.960421:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612663936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077004.960437:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077004.960446:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077004.960465:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612663936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077004.960481:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664064:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077004.960533:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077004.960559:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff8b10037df800 x1855007612664064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077004.960616:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612664064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077004.960642:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077004.960646:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077004.960650:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077004.960728:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077004.960758:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077004.960791:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff8b10037df480 x1855007612664192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077004.960808:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612664192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077004.960827:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077004.960831:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077004.960835:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077004.960842:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077004.960864:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077004.960876:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff8b10037dce00 x1855007612664320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077004.960890:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612664320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077004.960909:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077004.960913:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077004.960917:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077005.855763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077005.855903:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077006.879826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077006.879939:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077007.903799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077007.903902:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077008.927776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077008.927865:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077009.951714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077009.951844:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077010.079764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.079774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077010.079777:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.079785:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 3998 00000100:00080000:3.0:1769077010.079794:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.079799:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077010.079831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612664448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.079860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.079862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.079864:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.079867:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3998 00000100:00080000:3.0:1769077010.079885:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.079889:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077010.079897:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077010.079909:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612664576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.079918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.079921:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077010.079923:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.079928:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3998 00000100:00100000:0.0:1769077010.079931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612664448:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077010.079932:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.079938:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077010.079944:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077010.079954:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612664704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077010.079962:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077010.079965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077010.079967:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077010.079970:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 3998 00000100:00080000:3.0:1769077010.079973:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077010.079977:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077010.079981:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077010.080003:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612664832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077010.080015:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077010.080049:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff8b10037dea00 x1855007612664448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.080079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612664448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.080089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.080093:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077010.080097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612664448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077010.080104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077010.080121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077010.080130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff8b10037ddc00 x1855007612664576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.080140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612664576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.080152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.080155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077010.080157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077010.080176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.080190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077010.080197:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff8b10037df100 x1855007612664704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.080206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612664704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.080212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.080214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077010.080234:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077010.080239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077010.080254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077010.080263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff8b10037dd880 x1855007612664832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077010.080272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612664832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077010.080278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077010.080281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077010.080284:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612664832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077010.975760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077010.975850:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077011.999766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077011.999880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077013.023760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077013.023836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077014.047821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077014.047900:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077015.071792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077015.071964:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077015.199808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077015.199820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077015.199825:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077015.199834:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4003 00000100:00080000:0.0:1769077015.199846:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077015.199854:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077015.199892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612664960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077015.199924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077015.199928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077015.199930:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077015.199934:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4003 00000100:00080000:0.0:1769077015.199937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077015.199941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077015.199949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077015.199961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612665088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077015.199971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077015.199975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077015.199976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077015.199981:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4003 00000100:00080000:0.0:1769077015.199985:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077015.199988:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077015.199993:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077015.200003:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612665216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077015.200022:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612664960:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077015.200026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077015.200032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077015.200034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077015.200039:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4003 00000100:00080000:0.0:1769077015.200042:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077015.200046:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077015.200052:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077015.200066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612665344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077015.200100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077015.200178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff8b1006025180 x1855007612664960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.200223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612664960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.200238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.200244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077015.200249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612664960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077015.200259:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665088:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077015.200307:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077015.200320:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff8b1006027800 x1855007612665088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.200333:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612665088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.200351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.200354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077015.200357:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077015.200383:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077015.200412:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077015.200428:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff8b1006027100 x1855007612665216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.200448:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612665216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.200461:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.200467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077015.200472:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077015.200481:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077015.200509:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077015.200522:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff8b1006026680 x1855007612665344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.200542:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612665344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.200550:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.200553:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077015.200556:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077016.095704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077016.095756:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077017.119773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077017.119915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077018.143759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077018.143819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077019.167732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077019.167830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077020.191773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077020.191813:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077020.319784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.319793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077020.319797:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.319807:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4008 00000100:00080000:3.0:1769077020.319819:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.319826:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077020.319875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612665472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.319900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.319903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077020.319904:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.319908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4008 00000100:00080000:3.0:1769077020.319912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.319915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077020.319922:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077020.319932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612665600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.319940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077020.319945:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612665472:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077020.319945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077020.319949:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.319957:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4008 00000100:00080000:3.0:1769077020.319959:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.319962:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077020.319966:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077020.319975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612665728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.319981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.319984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077020.319985:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.319988:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4008 00000100:00080000:3.0:1769077020.319997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.320000:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077020.320002:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077020.320009:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077020.320013:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612665856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077020.320085:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff8b10037dc000 x1855007612665472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.320105:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612665472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.320114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.320117:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077020.320121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612665472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077020.320145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665600:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077020.320172:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077020.320185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff8b10037dd500 x1855007612665600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.320194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612665600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.320200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.320203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077020.320205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077020.320210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077020.320224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077020.320230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff8b10037ddf80 x1855007612665728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.320238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612665728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.320244:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.320246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077020.320248:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077020.320251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077020.320261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077020.320267:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff8b10037de300 x1855007612665856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.320274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612665856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.320279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.320281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077020.320283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612665856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077021.215781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077021.215877:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077022.239752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077022.239811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077023.263730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077023.263842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077024.287827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077024.287904:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077025.311749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077025.311786:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077025.439756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077025.439765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077025.439769:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077025.439777:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4013 00000100:00080000:0.0:1769077025.439787:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077025.439792:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077025.439827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612665984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077025.439851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077025.439854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077025.439855:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077025.439858:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4013 00000100:00080000:0.0:1769077025.439860:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077025.439863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077025.439870:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077025.439878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612666112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077025.439886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077025.439888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077025.439889:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077025.439892:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4013 00000100:00080000:0.0:1769077025.439894:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077025.439896:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077025.439900:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077025.439909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612666240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077025.439911:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612665984:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077025.439927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077025.439930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077025.439931:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077025.439934:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4013 00000100:00080000:0.0:1769077025.439937:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077025.439939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077025.439942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077025.439950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612666368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077025.439995:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077025.440031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff8b1006026680 x1855007612665984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077025.440058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612665984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077025.440070:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077025.440075:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077025.440079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612665984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077025.440087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077025.440109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077025.440122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff8b1006027100 x1855007612666112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077025.440136:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612666112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077025.440145:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077025.440148:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077025.440152:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077025.440178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077025.440212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077025.440225:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff8b1006027800 x1855007612666240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077025.440239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612666240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077025.440249:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077025.440252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077025.440256:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077025.440263:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077025.440293:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077025.440306:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff8b1006025180 x1855007612666368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077025.440319:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612666368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077025.440328:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077025.440332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077025.440335:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077026.335776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077026.335827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077027.359743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077027.359772:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077028.383737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077028.383831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077029.407820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077029.407936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077030.431744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077030.431828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077030.559801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.559810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077030.559814:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.559823:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4018 00000100:00080000:3.0:1769077030.559833:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.559839:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077030.559894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612666496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077030.559935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.559939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077030.559941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.559944:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4018 00000100:00080000:3.0:1769077030.559947:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.559950:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077030.559959:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077030.559972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612666624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077030.559982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077030.559986:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612666496:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077030.559986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077030.559990:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.559999:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4018 00000100:00080000:3.0:1769077030.560004:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.560007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077030.560011:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077030.560021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612666752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077030.560028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.560031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077030.560033:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.560036:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4018 00000100:00080000:3.0:1769077030.560046:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.560050:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077030.560053:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077030.560062:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612666880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077030.560096:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077030.560142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077030.560193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666880:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077031.455887:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077031.455910:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077032.479821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077032.479925:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077033.503723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077033.503796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769077033.631960:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769077033.631997:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769077033.632012:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077033.632041:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077033.632047:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077033.632053:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769077033.632106:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077033] req@ffff8b10037ded80 x1855007612666624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077033.632131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612666624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077033.632143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077033.632149:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077033.632153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077033.632162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077033] req@ffff8b10037de680 x1855007612666752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077033.632174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612666752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077033.632182:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077033.632185:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077033.632189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077033.632195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077033] req@ffff8b10037dc380 x1855007612666880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077033.632223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612666880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077033.632232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077033.632236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077033.632240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612666880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077033.632248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077033] req@ffff8b10037dd180 x1855007612666496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077033.632261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612666496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077033.632269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077033.632273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077033.632276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612666496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077034.527771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077034.527853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077035.551839:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077035.551875:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077035.679759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.679774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077035.679781:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.679808:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4023 00000100:00080000:3.0:1769077035.679823:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.679850:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077035.679913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612667008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.679950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.679954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077035.679956:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.679961:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4023 00000100:00080000:3.0:1769077035.679968:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.679978:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077035.680009:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077035.680025:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612667008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077035.680026:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612667136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.680050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.680057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077035.680061:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.680067:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4023 00000100:00080000:3.0:1769077035.680074:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.680083:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077035.680095:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077035.680122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612667264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.680134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.680138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077035.680140:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077035.680147:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077035.680149:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4023 00000100:00080000:3.0:1769077035.680157:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.680163:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077035.680170:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077035.680185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612667392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077035.680199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff8b10037dca80 x1855007612667008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077035.680225:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612667008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077035.680240:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077035.680246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077035.680251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612667008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077035.680290:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077035.680320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077035.680338:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff8b10037dd180 x1855007612667136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077035.680354:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612667136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077035.680364:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077035.680368:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077035.680372:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077035.680379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077035.680405:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077035.680419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff8b10037dc380 x1855007612667264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077035.680433:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612667264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077035.680443:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077035.680447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077035.680451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077035.680457:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077035.680483:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077035.680495:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff8b10037de680 x1855007612667392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077035.680515:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612667392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077035.680525:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077035.680529:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077035.680533:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077036.575720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077036.575814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077037.599753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077037.599834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077038.623720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077038.623881:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077039.647790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077039.647942:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077040.671757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077040.671882:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077040.799825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077040.799841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077040.799847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077040.799888:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4028 00000100:00080000:0.0:1769077040.799907:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077040.799919:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077040.799992:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612667520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077040.800027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077040.800031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077040.800033:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077040.800037:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4028 00000100:00080000:0.0:1769077040.800041:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077040.800046:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077040.800074:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077040.800087:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612667648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077040.800096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077040.800101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077040.800101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612667520:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077040.800105:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077040.800113:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4028 00000100:00080000:0.0:1769077040.800117:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077040.800122:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077040.800128:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077040.800143:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612667776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077040.800155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077040.800159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077040.800161:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077040.800166:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4028 00000100:00080000:0.0:1769077040.800174:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077040.800178:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077040.800185:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077040.800185:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077040.800202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612667904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077040.800228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff8b1006026300 x1855007612667520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077040.800247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612667520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077040.800273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077040.800277:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077040.800281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612667520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077040.800307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077040.800346:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077040.800359:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff8b1006025880 x1855007612667648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077040.800370:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612667648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077040.800378:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077040.800382:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077040.800387:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077040.800396:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077040.800424:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077040.800441:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff8b1006024000 x1855007612667776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077040.800458:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612667776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077040.800480:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077040.800483:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077040.800487:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077040.800494:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077040.800533:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077040.800551:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff8b1006026a00 x1855007612667904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077040.800570:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612667904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077040.800584:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077040.800589:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077040.800615:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612667904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077041.695849:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077041.695956:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077042.719776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077042.719949:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077043.743796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077043.743870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077044.767785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077044.767869:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077045.791749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077045.791829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077045.919737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077045.919746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077045.919750:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077045.919758:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4034 00000100:00080000:2.0:1769077045.919767:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077045.919773:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077045.919820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612668032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077045.919848:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077045.919852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077045.919854:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077045.919859:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4034 00000100:00080000:2.0:1769077045.919863:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077045.919868:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077045.919893:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077045.919903:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612668160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077045.919907:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612668032:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077045.919917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077045.919922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077045.919924:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077045.919927:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4034 00000100:00080000:2.0:1769077045.919929:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077045.919931:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077045.919935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077045.919945:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612668288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077045.919952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077045.919954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077045.919955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077045.919958:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4034 00000100:00080000:2.0:1769077045.919960:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077045.919962:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077045.919965:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077045.919972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612668416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077045.919985:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077045.920027:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077045] req@ffff8b1005d59880 x1855007612668032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077045.920042:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612668032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077045.920051:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077045.920054:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077045.920057:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612668032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077045.920075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668160:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077045.920093:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077045.920101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077045] req@ffff8b1005d59500 x1855007612668160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077045.920109:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612668160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077045.920115:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077045.920117:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077045.920119:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077045.920122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077045.920134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077045.920140:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077045] req@ffff8b1005d5ad80 x1855007612668288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077045.920152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612668288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077045.920157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077045.920159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077045.920161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077045.920164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077045.920182:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077045.920189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077045] req@ffff8b1005d58380 x1855007612668416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077045.920197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612668416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077045.920203:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077045.920205:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077045.920207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077046.815741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077046.815791:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077047.839741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077047.839808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077048.863750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077048.863827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077049.887726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077049.887779:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077050.911724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077050.911788:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077051.039759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.039769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077051.039774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.039782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4039 00000100:00080000:0.0:1769077051.039793:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.039799:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077051.039833:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612668544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077051.039860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.039864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.039866:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.039869:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4039 00000100:00080000:0.0:1769077051.039872:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.039875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077051.039884:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.039895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612668672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.039904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612668544:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077051.039923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.039926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077051.039928:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.039933:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4039 00000100:00080000:0.0:1769077051.039936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077051.039939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077051.039945:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.039956:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612668800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077051.039965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077051.039968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077051.039970:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077051.039973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4039 00000100:00080000:0.0:1769077051.039976:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077051.039981:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077051.039985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077051.039991:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077051.040001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612668928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077051.040026:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077051] req@ffff8b1006026d80 x1855007612668544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077051.040044:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612668544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077051.040054:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077051.040058:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077051.040062:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612668544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077051.040083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077051.040102:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077051.040112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077051] req@ffff8b1006027b80 x1855007612668672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077051.040123:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612668672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077051.040130:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077051.040133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077051.040136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077051.040140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077051.040155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077051.040162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077051] req@ffff8b1006025f80 x1855007612668800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077051.040172:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612668800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077051.040178:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077051.040180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077051.040183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077051.040187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077051.040200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077051.040207:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077051/real 1769077051] req@ffff8b1006024a80 x1855007612668928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077051.040217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612668928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077051.040223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077051.040225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077051.040227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612668928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077051.935728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077051.935782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077052.959726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077052.959758:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077053.983744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077053.983830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077055.007746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077055.007836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077056.031776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077056.031879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077056.159746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077056.159757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077056.159763:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077056.159773:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4044 00000100:00080000:0.0:1769077056.159796:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077056.159804:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077056.159843:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612669056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077056.159887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077056.159891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077056.159893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077056.159899:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4044 00000100:00080000:0.0:1769077056.159903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077056.159907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077056.159929:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077056.159944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612669184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077056.159957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077056.159961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077056.159964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077056.159971:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4044 00000100:00100000:2.0:1769077056.159972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612669056:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077056.159977:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077056.159984:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077056.159991:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077056.160007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612669312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077056.160020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077056.160025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077056.160027:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077056.160032:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4044 00000100:00080000:0.0:1769077056.160037:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077056.160037:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077056.160047:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077056.160055:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077056.160083:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612669440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077056.160087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff8b1006024e00 x1855007612669056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.160125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612669056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.160142:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.160145:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077056.160149:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612669056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077056.160154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669184:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077056.160170:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077056.160178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff8b1006024380 x1855007612669184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.160187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612669184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.160193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.160195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077056.160197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077056.160215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077056.160228:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077056.160240:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff8b1006027480 x1855007612669312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.160254:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612669312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.160270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.160274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077056.160278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077056.160285:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077056.160325:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077056.160339:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff8b1006024700 x1855007612669440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.160349:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612669440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.160366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.160369:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077056.160371:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077057.055755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077057.055825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077058.079795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077058.079868:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077059.103784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077059.103879:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077060.127713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077060.127760:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077061.151747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077061.151814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077061.279814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077061.279851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077061.279856:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077061.279878:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4049 00000100:00080000:3.0:1769077061.279889:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077061.279896:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077061.279935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612669568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077061.279978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077061.279980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077061.279982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077061.279985:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4049 00000100:00080000:3.0:1769077061.279987:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077061.279990:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077061.279997:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077061.280007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612669696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077061.280028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077061.280031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077061.280032:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077061.280035:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4049 00000100:00080000:3.0:1769077061.280037:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077061.280041:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077061.280046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612669568:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769077061.280046:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077061.280058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612669824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077061.280065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077061.280067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077061.280069:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077061.280071:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4049 00000100:00080000:3.0:1769077061.280073:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077061.280075:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077061.280087:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077061.280096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612669952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077061.280133:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077061.280204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff8b10037ded80 x1855007612669568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.280269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612669568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.280284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.280300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077061.280321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612669568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077061.280329:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077061.280393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077061.280405:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff8b10037de300 x1855007612669696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.280415:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612669696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.280421:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.280424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077061.280426:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077061.280466:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077061.280488:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077061.280498:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff8b10037ddf80 x1855007612669824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.280509:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612669824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.280527:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.280530:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077061.280543:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077061.280548:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077061.280565:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077061.280575:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff8b10037dd500 x1855007612669952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.280586:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612669952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.280593:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.280595:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077061.280598:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612669952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077062.175734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077062.175789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077063.199754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077063.199804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077064.223807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077064.223859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077065.247740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077065.247831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077066.271736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077066.271795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077066.399739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077066.399748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077066.399751:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077066.399759:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4054 00000100:00080000:3.0:1769077066.399769:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077066.399775:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077066.399811:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612670080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077066.399841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077066.399844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077066.399846:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077066.399849:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4054 00000100:00080000:3.0:1769077066.399852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077066.399855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077066.399863:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077066.399873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612670208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077066.399881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077066.399883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077066.399885:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077066.399888:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4054 00000100:00080000:3.0:1769077066.399890:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077066.399893:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077066.399896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077066.399906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612670336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077066.399907:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612670080:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077066.399921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077066.399923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077066.399925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077066.399927:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4054 00000100:00080000:3.0:1769077066.399930:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077066.399933:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077066.399936:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077066.399946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612670464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077066.399972:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077066.400003:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff8b10037dc000 x1855007612670080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077066.400017:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612670080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077066.400024:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077066.400027:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077066.400030:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612670080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077066.400035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077066.400050:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077066.400058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff8b10037dd880 x1855007612670208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077066.400067:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612670208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077066.400072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077066.400074:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077066.400076:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077066.400094:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077066.400106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077066.400112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff8b10037df100 x1855007612670336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077066.400120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612670336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077066.400125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077066.400127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077066.400129:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077066.400133:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077066.400142:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077066.400149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff8b10037ddc00 x1855007612670464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077066.400156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612670464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077066.400166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077066.400168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077066.400170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077067.295752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077067.295788:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077068.319719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077068.319776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077069.343758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077069.343768:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077070.367748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077070.367973:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077071.391787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077071.391841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077071.519774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077071.519786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077071.519791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077071.519802:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4059 00000100:00080000:3.0:1769077071.519817:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077071.519827:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077071.519880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612670592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077071.519912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077071.519916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077071.519918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077071.519926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4059 00000100:00080000:3.0:1769077071.519934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077071.519940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077071.519954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077071.519957:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612670592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077071.519989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612670720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077071.520006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077071.520010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077071.520012:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077071.520016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4059 00000100:00080000:3.0:1769077071.520022:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077071.520031:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769077071.520032:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077071.520065:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff8b10037dea00 x1855007612670592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077071.520079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612670592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077071.520089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077071.520092:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077071.520096:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612670592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077071.520118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670720:192.168.204.102@tcp:38:kworker.0 00000008:00100000:3.0:1769077071.520119:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077071.520155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077071.520155:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612670848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077071.520165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff8b10037dce00 x1855007612670720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077071.520178:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612670720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077071.520184:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077071.520200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077071.520203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670720:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769077071.520356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077071.520361:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077071.520365:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077071.520371:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4059 00000100:00080000:3.0:1769077071.520376:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077071.520383:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077071.520388:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670848:192.168.204.102@tcp:8:kworker.0 00000008:00100000:3.0:1769077071.520390:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077071.520424:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612670976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077071.520436:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077071.520457:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff8b10037df480 x1855007612670848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077071.520467:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612670848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077071.520473:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077071.520475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077071.520477:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077071.520488:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077071.520503:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077071.520515:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff8b10037df800 x1855007612670976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077071.520525:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612670976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077071.520533:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077071.520536:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077071.520538:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612670976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077072.415712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077072.415792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077073.439745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077073.439823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077074.463744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077074.463809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077075.487735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077075.487789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077076.511758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077076.511844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077076.639824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077076.639835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077076.639840:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077076.639850:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4064 00000100:00080000:1.0:1769077076.639873:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077076.639880:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077076.639923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612671104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077076.639968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077076.639973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077076.639975:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077076.639979:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4064 00000100:00080000:1.0:1769077076.639983:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077076.639986:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077076.640007:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077076.640018:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612671232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077076.640028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077076.640032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077076.640034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077076.640037:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4064 00000100:00080000:1.0:1769077076.640040:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077076.640044:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077076.640050:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077076.640060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612671360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077076.640071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077076.640074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077076.640076:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077076.640083:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612671104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077076.640084:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4064 00000100:00080000:1.0:1769077076.640092:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077076.640098:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077076.640134:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077076.640175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612671488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077076.640205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077076.640246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff8b1006137480 x1855007612671104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077076.640259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612671104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077076.640267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077076.640271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077076.640274:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612671104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077076.640279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077076.640292:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077076.640300:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff8b1006134a80 x1855007612671232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077076.640308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612671232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077076.640314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077076.640316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077076.640318:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077076.640335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077076.640357:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077076.640363:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff8b1006134e00 x1855007612671360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077076.640372:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612671360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077076.640377:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077076.640379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077076.640381:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077076.640384:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077076.640394:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077076.640400:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff8b1006134700 x1855007612671488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077076.640419:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612671488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077076.640424:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077076.640426:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077076.640427:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077077.535730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077077.535789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077078.559741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077078.559805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077079.583751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077079.583808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077080.607757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077080.607912:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077081.631718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077081.631785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077081.759741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077081.759751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077081.759754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077081.759762:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4069 00000100:00080000:1.0:1769077081.759772:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077081.759777:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077081.759812:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612671616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077081.759835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077081.759838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077081.759839:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077081.759842:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4069 00000100:00080000:1.0:1769077081.759845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077081.759847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077081.759854:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077081.759863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612671744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077081.759873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077081.759877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077081.759880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077081.759882:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612671616:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077081.759886:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4069 00000100:00080000:1.0:1769077081.759893:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077081.759895:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077081.759899:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077081.759909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612671872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077081.759916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077081.759918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077081.759919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077081.759922:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4069 00000100:00080000:1.0:1769077081.759936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077081.759938:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077081.759941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077081.759944:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077081.759953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612672000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077081.759977:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff8b1006137100 x1855007612671616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077081.759998:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612671616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077081.760005:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077081.760008:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077081.760011:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612671616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077081.760062:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671744:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077081.760088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077081.760101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff8b1006136680 x1855007612671744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077081.760110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612671744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077081.760119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077081.760123:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077081.760126:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077081.760132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077081.760155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077081.760181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff8b1006136d80 x1855007612671872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077081.760194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612671872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077081.760210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077081.760213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077081.760217:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612671872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077081.760224:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077081.760244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077081.760253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff8b1006135500 x1855007612672000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077081.760261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612672000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077081.760267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077081.760269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077081.760271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077082.655710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077082.655782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077083.679727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077083.679800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077084.703740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077084.703791:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077085.727709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077085.727771:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077086.751770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077086.751834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077086.879754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077086.879765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077086.879769:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077086.879777:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4074 00000100:00080000:2.0:1769077086.879788:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077086.879794:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077086.879827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612672128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077086.879850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077086.879853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077086.879854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077086.879857:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4074 00000100:00080000:2.0:1769077086.879859:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077086.879862:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077086.879868:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077086.879878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612672256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077086.879886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077086.879888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077086.879890:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077086.879894:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4074 00000100:00100000:0.0:1769077086.879896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612672128:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077086.879896:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077086.879901:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077086.879908:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077086.879917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612672384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077086.879923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077086.879926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077086.879942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077086.879945:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4074 00000100:00080000:2.0:1769077086.879948:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077086.879951:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077086.879955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077086.879964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612672512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077086.879981:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077086.880015:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff8b1005d5b100 x1855007612672128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077086.880033:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612672128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077086.880051:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077086.880055:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077086.880059:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612672128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077086.880089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077086.880112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077086.880125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff8b1005d5aa00 x1855007612672256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077086.880138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612672256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077086.880148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077086.880151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077086.880155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077086.880162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077086.880181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077086.880192:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff8b1005d58380 x1855007612672384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077086.880204:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612672384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077086.880212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077086.880216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077086.880219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077086.880226:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077086.880243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077086.880253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff8b1005d5ad80 x1855007612672512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077086.880265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612672512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077086.880273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077086.880276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077086.880279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077087.775770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077087.775790:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077088.799744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077088.799816:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077089.823741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077089.823808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077090.847759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077090.847786:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077091.871728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077091.871840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077091.999749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077091.999760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077091.999764:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077091.999774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4080 00000100:00080000:1.0:1769077091.999785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077091.999792:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077091.999825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612672640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077091.999877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077091.999882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077091.999884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077091.999888:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4080 00000100:00080000:1.0:1769077091.999892:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077091.999898:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077091.999908:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077091.999916:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612672640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077091.999923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612672768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077091.999936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077091.999940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077091.999942:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077091.999946:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4080 00000100:00080000:1.0:1769077091.999950:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077091.999955:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077091.999960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077091.999972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612672896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077091.999981:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077091.999990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077091.999993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077091.999996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077092.000001:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4080 00000100:00080000:1.0:1769077092.000007:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077092.000011:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077092.000014:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff8b1006135880 x1855007612672640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769077092.000019:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077092.000039:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612672640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077092.000040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612673024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077092.000061:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.000067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077092.000071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612672640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077092.000091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077092.000108:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077092.000117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077092] req@ffff8b1006137b80 x1855007612672768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.000125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612672768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.000131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.000133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077092.000135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077092.000139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.000150:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077092.000170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077092] req@ffff8b1006135c00 x1855007612672896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.000184:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612672896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.000193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.000197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077092.000200:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612672896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077092.000207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077092.000226:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077092.000236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077092] req@ffff8b1006137800 x1855007612673024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077092.000249:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612673024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077092.000254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077092.000256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077092.000259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077092.895728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077092.895789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077093.919751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077093.919855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077094.943747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077094.943760:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077095.967758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077095.967827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769077096.991852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077096.991863:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:1.0:1769077097.119863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077097.119874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077097.119880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077097.119889:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4085 00000100:00080000:1.0:1769077097.119901:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077097.119908:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077097.119948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612673152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077097.119980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077097.119984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077097.119986:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077097.119990:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4085 00000100:00080000:1.0:1769077097.119993:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077097.119997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077097.120005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077097.120017:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612673280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077097.120028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077097.120035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077097.120037:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077097.120041:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4085 00000100:00080000:1.0:1769077097.120044:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077097.120052:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077097.120054:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612673152:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769077097.120058:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077097.120073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612673408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077097.120084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077097.120088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077097.120089:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077097.120093:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4085 00000100:00080000:1.0:1769077097.120098:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077097.120102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077097.120107:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077097.120119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612673536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077097.120171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077097.120209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077097.120234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077098.015762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077098.015817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077099.039780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077099.039848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077100.063774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077100.063849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077100.192302:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077100.192394:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077100.192434:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077100.192499:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077100.192514:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077100.192526:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769077100.193003:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077100] req@ffff8b1006136300 x1855007612673408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077100.193075:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612673408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077100.193097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077100.193105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077100.193114:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077100.193128:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077100] req@ffff8b1006137800 x1855007612673536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077100.193148:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612673536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077100.193163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077100.193168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077100.193172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077100.193182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077100] req@ffff8b1006135180 x1855007612673152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077100.193197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612673152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077100.193208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077100.193213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077100.193218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612673152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077100.193228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077100] req@ffff8b1006136a00 x1855007612673280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077100.193252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612673280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077100.193263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077100.193268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077100.193273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077101.087941:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077101.088285:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077102.111783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077102.111985:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077102.239858:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077102.239867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077102.239873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077102.239885:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4090 00000100:00080000:3.0:1769077102.239895:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077102.239902:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077102.239932:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612673664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077102.239981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077102.239984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077102.239986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077102.239990:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4090 00000100:00080000:3.0:1769077102.239992:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077102.239995:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077102.240002:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077102.240035:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612673792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077102.240048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077102.240051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077102.240052:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077102.240058:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4090 00000100:00080000:3.0:1769077102.240062:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077102.240070:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077102.240071:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612673664:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769077102.240075:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077102.240089:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612673920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077102.240096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077102.240098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077102.240099:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077102.240101:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4090 00000100:00080000:3.0:1769077102.240104:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077102.240113:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077102.240116:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077102.240126:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612674048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077102.240198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077102.240270:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff8b10037dc700 x1855007612673664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.240295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612673664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.240310:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.240316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077102.240321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612673664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077102.240357:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077102.240389:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077102.240405:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff8b10037df800 x1855007612673792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.240420:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612673792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.240430:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.240434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077102.240438:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077102.240447:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077102.240481:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077102.240495:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff8b10037df480 x1855007612673920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.240508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612673920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.240518:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.240522:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077102.240527:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612673920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077102.240535:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077102.240560:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077102.240572:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff8b10037dce00 x1855007612674048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.240587:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612674048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.240597:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.240601:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077102.240605:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077103.135755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077103.135823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077104.159754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077104.159825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077105.183733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077105.183807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077106.207770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077106.207800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077107.231709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077107.231793:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077107.359720:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077107.359728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077107.359733:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077107.359744:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4095 00000100:00080000:3.0:1769077107.359756:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077107.359764:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077107.359813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612674176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077107.359841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077107.359845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077107.359848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077107.359853:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4095 00000100:00080000:3.0:1769077107.359859:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077107.359863:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077107.359874:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077107.359905:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612674176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077107.359911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612674304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077107.359924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077107.359928:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077107.359930:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077107.359936:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4095 00000100:00080000:3.0:1769077107.359940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077107.359945:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077107.359950:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077107.359971:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077107.360008:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff8b10037dea00 x1855007612674176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.360022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612674432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077107.360026:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612674176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.360034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077107.360037:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.360037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077107.360039:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077107.360041:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00080000:3.0:1769077107.360042:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4095 00000100:00100000:2.0:1769077107.360046:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612674176:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077107.360046:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077107.360050:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077107.360056:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077107.360067:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077107.360067:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612674560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077107.360091:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077107.360105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff8b10037ddc00 x1855007612674304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077107.360119:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612674304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077107.360126:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077107.360129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077107.360132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077107.360139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077107.360176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077107.360189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff8b10037df100 x1855007612674432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077107.360201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612674432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077107.360209:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077107.360212:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077107.360215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077107.360230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077107.360252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077107.360265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff8b10037dd880 x1855007612674560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077107.360277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612674560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077107.360285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077107.360289:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077107.360292:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077108.255735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077108.255810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077109.279718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077109.279780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077110.303742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077110.303833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077111.327730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077111.327787:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077112.351744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077112.351808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077112.479713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.479722:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077112.479725:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.479735:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4100 00000100:00080000:3.0:1769077112.479746:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.479753:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077112.479791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612674688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077112.479815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.479825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077112.479827:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.479830:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4100 00000100:00080000:3.0:1769077112.479834:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.479838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077112.479846:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077112.479851:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612674688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077112.479863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612674816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077112.479876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.479879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077112.479881:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.479886:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4100 00000100:00080000:3.0:1769077112.479902:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.479907:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077112.479912:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077112.479918:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077112.479926:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612674944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077112.479935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.479938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077112.479939:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.479942:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4100 00000100:00080000:3.0:1769077112.479946:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.479949:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077112.479954:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077112.479955:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077112] req@ffff8b10037dc000 x1855007612674688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077112.479965:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612675072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077112.479977:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612674688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077112.479987:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077112.479991:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077112.479994:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612674688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077112.480012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674816:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077112.480030:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077112.480038:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077112] req@ffff8b10037dd500 x1855007612674816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077112.480047:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612674816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077112.480052:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077112.480055:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077112.480057:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077112.480061:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077112.480073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077112.480079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077112] req@ffff8b10037ddf80 x1855007612674944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077112.480087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612674944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077112.480097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077112.480098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077112.480100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612674944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077112.480104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077112.480114:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077112.480120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077112] req@ffff8b10037de300 x1855007612675072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077112.480127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612675072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077112.480132:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077112.480134:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077112.480136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077113.375752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077113.375820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077114.399755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077114.399815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077115.423736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077115.423796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077116.447758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077116.447815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077117.471755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077117.471821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077117.599741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.599753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077117.599758:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.599768:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4105 00000100:00080000:3.0:1769077117.599781:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.599788:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077117.599832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612675200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.599862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.599866:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077117.599867:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.599871:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4105 00000100:00080000:3.0:1769077117.599873:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.599875:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077117.599885:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077117.599895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612675328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.599909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077117.599914:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612675200:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077117.599914:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077117.599916:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.599924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4105 00000100:00080000:3.0:1769077117.599927:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.599930:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077117.599933:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077117.599942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612675456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.599948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.599951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077117.599953:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.599955:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4105 00000100:00080000:3.0:1769077117.599958:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.599961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077117.599964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077117.599971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612675584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077117.599994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077117.600030:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff8b10037ded80 x1855007612675200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077117.600046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612675200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077117.600055:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077117.600059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077117.600063:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612675200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077117.600085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077117.600100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077117.600109:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff8b10037de680 x1855007612675328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077117.600118:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612675328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077117.600124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077117.600127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077117.600129:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077117.600133:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077117.600145:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077117.600152:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff8b10037dc380 x1855007612675456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077117.600160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612675456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077117.600165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077117.600167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077117.600169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077117.600173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077117.600184:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077117.600191:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff8b10037dd180 x1855007612675584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077117.600199:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612675584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077117.600218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077117.600220:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077117.600223:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077118.495728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077118.495844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077119.519768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077119.519807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077120.543734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077120.543798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077121.567752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077121.567853:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077122.591777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077122.591796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077122.719781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.719795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077122.719801:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.719813:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4110 00000100:00080000:3.0:1769077122.719826:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.719835:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077122.719884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612675712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.719923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.719928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077122.719931:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.719936:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4110 00000100:00080000:3.0:1769077122.719940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.719945:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077122.719968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077122.719987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612675840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.720002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.720008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077122.720011:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077122.720034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612675712:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077122.720047:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4110 00000100:00080000:3.0:1769077122.720051:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.720055:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077122.720060:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077122.720072:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612675968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.720079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.720082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077122.720083:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.720086:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4110 00000100:00080000:3.0:1769077122.720088:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.720091:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077122.720094:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077122.720102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612676096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077122.720145:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077122.720197:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff8b10037dca80 x1855007612675712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077122.720228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612675712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077122.720238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077122.720242:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077122.720246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612675712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077122.720290:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675840:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077122.720313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077122.720339:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff8b10037dd180 x1855007612675840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077122.720375:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612675840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077122.720387:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077122.720392:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077122.720396:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077122.720406:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077122.720468:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077122.720483:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff8b10037dc380 x1855007612675968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077122.720497:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612675968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077122.720506:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077122.720509:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077122.720512:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612675968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077122.720519:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077122.720540:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077122.720550:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff8b10037de680 x1855007612676096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077122.720561:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612676096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077122.720576:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077122.720578:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077122.720581:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077123.615776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077123.615824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077124.639729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077124.639837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077125.663811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077125.663964:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077126.687822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077126.687888:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077127.711748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077127.711811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077127.839728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.839740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077127.839745:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.839755:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4115 00000100:00080000:3.0:1769077127.839768:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.839776:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077127.839821:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612676224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077127.839849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.839853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077127.839855:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.839858:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4115 00000100:00080000:3.0:1769077127.839861:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.839864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077127.839873:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077127.839883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612676352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077127.839891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.839894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077127.839895:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.839898:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4115 00000100:00080000:3.0:1769077127.839900:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.839903:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077127.839906:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077127.839915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612676224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077127.839915:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612676480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077127.839929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.839931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077127.839933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.839935:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4115 00000100:00080000:3.0:1769077127.839937:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.839939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077127.839943:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077127.839952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612676608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077127.840004:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077127.840043:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff8b10037ded80 x1855007612676224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077127.840060:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612676224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077127.840069:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077127.840075:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077127.840079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612676224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077127.840088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077127.840112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077127.840136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff8b10037de300 x1855007612676352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077127.840147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612676352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077127.840153:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077127.840156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077127.840158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077127.840182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077127.840199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077127.840210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff8b10037ddf80 x1855007612676480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077127.840223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612676480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077127.840232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077127.840236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077127.840254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077127.840262:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077127.840282:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077127.840291:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff8b10037dd500 x1855007612676608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077127.840299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612676608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077127.840305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077127.840308:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077127.840310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077128.735704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077128.735800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077129.759769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077129.759862:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077130.783745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077130.783766:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077131.807720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077131.807796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077132.831750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077132.831851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077132.959737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077132.959746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077132.959750:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077132.959758:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4121 00000100:00080000:3.0:1769077132.959769:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077132.959774:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077132.959807:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612676736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077132.959831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077132.959834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077132.959835:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077132.959838:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4121 00000100:00080000:3.0:1769077132.959840:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077132.959843:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077132.959850:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077132.959857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612676864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077132.959871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077132.959873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077132.959874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077132.959877:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4121 00000100:00080000:3.0:1769077132.959878:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077132.959881:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077132.959884:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077132.959892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612676992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077132.959894:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612676736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077132.959903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077132.959907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077132.959908:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077132.959910:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4121 00000100:00080000:3.0:1769077132.959912:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077132.959915:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077132.959919:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077132.959931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612677120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077132.959970:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077132.960017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff8b10037dc000 x1855007612676736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077132.960037:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612676736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077132.960049:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077132.960054:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077132.960058:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612676736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077132.960065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077132.960090:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077132.960102:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff8b10037dd880 x1855007612676864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077132.960116:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612676864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077132.960124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077132.960128:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077132.960132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077132.960193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077132.960219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077132.960239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff8b10037df100 x1855007612676992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077132.960261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612676992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077132.960271:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077132.960274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077132.960278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612676992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077132.960285:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077132.960308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077132.960322:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff8b10037ddc00 x1855007612677120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077132.960335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612677120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077132.960345:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077132.960349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077132.960352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077133.855784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077133.855872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077134.879760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077134.879835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077135.903790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077135.903821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077136.927740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077136.927801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077137.951720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077137.951799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077138.079761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077138.079770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077138.079773:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077138.079781:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4126 00000100:00080000:2.0:1769077138.079790:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077138.079796:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077138.079828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612677248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077138.079856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077138.079860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077138.079862:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077138.079867:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4126 00000100:00080000:2.0:1769077138.079870:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077138.079874:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077138.079884:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077138.079896:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612677376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077138.079906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077138.079908:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612677248:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077138.079910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077138.079912:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077138.079915:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4126 00000100:00080000:2.0:1769077138.079918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077138.079921:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077138.079926:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077138.079934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612677504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077138.079942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077138.079945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077138.079948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077138.079951:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4126 00000100:00080000:2.0:1769077138.079954:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077138.079957:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077138.079961:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077138.079966:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077138.079970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612677632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077138.079997:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff8b1005d59500 x1855007612677248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077138.080015:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612677248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077138.080026:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077138.080031:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077138.080035:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612677248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077138.080060:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077138.080157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077138.080168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff8b1005d59880 x1855007612677376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077138.080177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612677376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077138.080190:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077138.080194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077138.080201:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077138.080207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077138.080229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077138.080241:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff8b1005d5a300 x1855007612677504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077138.080253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612677504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077138.080261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077138.080264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077138.080268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077138.080274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077138.080295:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077138.080307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff8b1005d5bb80 x1855007612677632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077138.080321:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612677632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077138.080326:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077138.080328:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077138.080330:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077138.975756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077138.975817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077139.999735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077139.999801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077141.023764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077141.023857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077142.047772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077142.047782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077143.071750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077143.071805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077143.199744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077143.199753:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077143.199756:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077143.199766:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4131 00000100:00080000:2.0:1769077143.199775:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077143.199782:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077143.199813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612677760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077143.199843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077143.199847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077143.199848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077143.199851:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4131 00000100:00080000:2.0:1769077143.199854:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077143.199857:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077143.199864:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077143.199875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612677888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077143.199884:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077143.199887:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077143.199888:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077143.199892:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4131 00000100:00080000:2.0:1769077143.199895:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077143.199897:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077143.199903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612677760:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769077143.199904:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077143.199918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612678016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077143.199926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077143.199929:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077143.199930:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077143.199933:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4131 00000100:00080000:2.0:1769077143.199935:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077143.199938:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077143.199941:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077143.199949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612678144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077143.199977:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077143.200027:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff8b1005d5b480 x1855007612677760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077143.200049:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612677760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077143.200067:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077143.200072:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077143.200077:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612677760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077143.200085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077143.200111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077143.200125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff8b1005d58e00 x1855007612677888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077143.200141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612677888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077143.200152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077143.200156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077143.200162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612677888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077143.200192:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077143.200220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077143.200239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff8b1005d58000 x1855007612678016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077143.200258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612678016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077143.200270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077143.200275:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077143.200280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077143.200289:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077143.200316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077143.200334:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff8b1005d58a80 x1855007612678144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077143.200354:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612678144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077143.200364:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077143.200368:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077143.200374:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077144.095736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077144.095789:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077145.119736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077145.119830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077146.143710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077146.143785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077147.167749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077147.167817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077148.191704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077148.191818:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077148.319766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077148.319777:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077148.319781:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077148.319791:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4136 00000100:00080000:0.0:1769077148.319802:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077148.319809:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077148.319845:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612678272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077148.319873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077148.319877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077148.319879:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077148.319884:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4136 00000100:00080000:0.0:1769077148.319888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077148.319892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077148.319914:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077148.319928:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612678272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077148.319930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612678400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077148.319946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077148.319949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077148.319951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077148.319956:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4136 00000100:00080000:0.0:1769077148.319960:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077148.319965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077148.319970:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077148.319981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612678528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077148.319989:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077148.319997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077148.320001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077148.320004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077148.320009:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4136 00000100:00080000:0.0:1769077148.320013:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077148.320018:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077148.320022:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff8b1006024700 x1855007612678272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077148.320025:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077148.320043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612678656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077148.320045:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612678272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077148.320071:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077148.320076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077148.320079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612678272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077148.320098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678400:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077148.320113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077148.320121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff8b1006027480 x1855007612678400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077148.320130:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612678400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077148.320136:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077148.320138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077148.320140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077148.320144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077148.320154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077148.320160:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff8b1006024380 x1855007612678528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077148.320168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612678528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077148.320173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077148.320175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077148.320177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077148.320180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077148.320190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077148.320195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff8b1006024e00 x1855007612678656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077148.320203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612678656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077148.320213:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077148.320215:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077148.320217:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678656:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077149.215843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077149.215866:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077150.239784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077150.239914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077151.263745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077151.263888:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077152.287782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077152.287822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077153.311772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077153.311831:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077153.439796:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077153.439804:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077153.439840:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077153.439849:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4141 00000100:00080000:2.0:1769077153.439859:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077153.439864:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077153.439907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612678784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077153.439933:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077153.439936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077153.439937:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077153.439940:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4141 00000100:00080000:2.0:1769077153.439942:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077153.439945:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077153.439952:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077153.439961:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612678912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077153.439968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077153.439971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077153.439972:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077153.439975:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4141 00000100:00080000:2.0:1769077153.439977:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077153.439979:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077153.439982:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077153.439989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612679040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077153.439996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077153.439999:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077153.440000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077153.440004:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4141 00000100:00080000:2.0:1769077153.440007:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077153.440008:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612678784:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077153.440010:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077153.440018:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077153.440027:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612679168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077153.440068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077153.440096:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff8b1005d5b800 x1855007612678784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077153.440110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612678784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077153.440118:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077153.440121:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077153.440124:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612678784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077153.440129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077153.440142:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077153.440150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff8b1005d59180 x1855007612678912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077153.440158:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612678912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077153.440163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077153.440165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077153.440167:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612678912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077153.440195:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077153.440212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077153.440219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff8b1005d58a80 x1855007612679040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077153.440227:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612679040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077153.440233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077153.440235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077153.440237:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077153.440240:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077153.440251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077153.440257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff8b1005d58000 x1855007612679168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077153.440264:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612679168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077153.440279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077153.440281:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077153.440283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077154.335814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077154.335923:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077155.359753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077155.359807:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077156.383932:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077156.383956:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077157.407755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077157.407808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077158.431793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077158.431860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077158.559764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077158.559774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077158.559777:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077158.559784:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4146 00000100:00080000:3.0:1769077158.559794:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077158.559800:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077158.559832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612679296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077158.559856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077158.559859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077158.559860:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077158.559863:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4146 00000100:00080000:3.0:1769077158.559865:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077158.559868:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077158.559875:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077158.559884:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612679424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077158.559893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077158.559896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077158.559897:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077158.559902:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4146 00000100:00100000:1.0:1769077158.559903:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612679296:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077158.559904:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077158.559911:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077158.559916:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077158.559924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612679552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077158.559931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077158.559934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077158.559935:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077158.559937:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4146 00000100:00080000:3.0:1769077158.559940:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077158.559942:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077158.559945:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077158.559951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612679680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077158.559966:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077158.559996:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff8b10037dea00 x1855007612679296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077158.560011:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612679296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077158.560019:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077158.560022:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077158.560025:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612679296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077158.560047:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077158.560070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077158.560083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff8b10037dce00 x1855007612679424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077158.560097:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612679424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077158.560105:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077158.560109:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077158.560112:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077158.560119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077158.560141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077158.560153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff8b10037df480 x1855007612679552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077158.560166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612679552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077158.560191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077158.560199:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077158.560203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077158.560210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077158.560232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077158.560245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff8b10037df800 x1855007612679680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077158.560257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612679680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077158.560266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077158.560269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077158.560273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077159.455792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077159.455864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077160.479764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077160.479827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077161.503730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077161.503824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077162.527752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077162.527835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077163.551850:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077163.551865:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077163.679797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.679808:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077163.679812:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.679822:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4151 00000100:00080000:2.0:1769077163.679833:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.679840:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077163.679878:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612679808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.679924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.679927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077163.679929:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.679931:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4151 00000100:00080000:2.0:1769077163.679934:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.679937:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077163.679944:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077163.679953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612679936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.679961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.679963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077163.679965:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.679968:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4151 00000100:00080000:2.0:1769077163.679970:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.679973:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077163.679976:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077163.679986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612680064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077163.679988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612679808:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077163.679999:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.680002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077163.680003:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.680006:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4151 00000100:00080000:2.0:1769077163.680008:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.680011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077163.680014:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077163.680022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612680192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077163.680075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077163.680106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077163.680121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077164.575861:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077164.575936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077165.599731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077165.599802:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077166.623751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077166.623806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077166.751946:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769077166.752029:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077166.752047:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077166.752087:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077166.752112:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077166.752121:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769077166.752156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077166] req@ffff8b1005d5bb80 x1855007612680064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077166.752188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612680064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077166.752201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077166.752206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077166.752210:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077166.752218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077166] req@ffff8b1005d5a300 x1855007612680192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077166.752231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612680192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077166.752239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077166.752242:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077166.752246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077166.752253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077166] req@ffff8b1005d58e00 x1855007612679808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077166.752265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612679808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077166.752274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077166.752277:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077166.752280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612679808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077166.752287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077166] req@ffff8b1005d5b480 x1855007612679936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077166.752299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612679936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077166.752307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077166.752311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077166.752314:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612679936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077167.647735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077167.647795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077168.671750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077168.671815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077168.799828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.799836:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077168.799839:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.799848:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4156 00000100:00080000:2.0:1769077168.799858:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.799864:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077168.799895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612680320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.799930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.799933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077168.799934:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.799937:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4156 00000100:00080000:2.0:1769077168.799940:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.799942:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077168.799949:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077168.799957:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612680448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.799963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.799966:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077168.799967:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.799969:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4156 00000100:00080000:2.0:1769077168.799971:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.799974:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077168.799977:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077168.799983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612680576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.799989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.799991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077168.799993:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.799995:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4156 00000100:00080000:2.0:1769077168.799997:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.799999:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077168.800002:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077168.800007:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612680704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077168.800033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612680320:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077168.800103:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077168.800138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff8b1005d59880 x1855007612680320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077168.800155:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612680320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077168.800166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077168.800171:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077168.800175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612680320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077168.800182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077168.800205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077168.800219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff8b1005d59500 x1855007612680448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077168.800232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612680448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077168.800241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077168.800244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077168.800248:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077168.800255:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077168.800278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077168.800289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff8b1005d5ad80 x1855007612680576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077168.800300:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612680576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077168.800307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077168.800311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077168.800314:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077168.800319:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077168.800336:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077168.800345:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff8b1005d58380 x1855007612680704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077168.800356:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612680704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077168.800364:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077168.800367:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077168.800370:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077169.695724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077169.695774:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077170.719777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077170.719812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077171.743784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077171.743859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077172.767729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077172.767841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077173.791783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077173.791843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077173.919766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.919779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077173.919785:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.919795:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4162 00000100:00080000:3.0:1769077173.919808:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.919815:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077173.919860:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612680832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.919900:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.919905:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077173.919907:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.919912:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4162 00000100:00080000:3.0:1769077173.919916:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.919922:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077173.919934:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077173.919950:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612680960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.919959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.919962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077173.919964:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.919967:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4162 00000100:00080000:3.0:1769077173.919970:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.919973:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077173.919977:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077173.919984:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612680832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077173.919988:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612681088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.919998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.920001:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077173.920002:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.920004:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4162 00000100:00080000:3.0:1769077173.920008:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.920011:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077173.920014:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077173.920022:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612681216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077173.920065:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077173.920103:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff8b10037dc700 x1855007612680832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.920120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612680832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.920129:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.920133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077173.920136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612680832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077173.920142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077173.920156:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077173.920165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff8b10037df800 x1855007612680960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.920173:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612680960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.920179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.920181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077173.920183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612680960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077173.920205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077173.920217:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077173.920223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff8b10037df480 x1855007612681088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.920231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612681088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.920241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.920243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077173.920245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077173.920248:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077173.920259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077173.920265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff8b10037dce00 x1855007612681216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.920271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612681216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.920276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.920278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077173.920280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077174.815747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077174.815942:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077175.839819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077175.839885:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077176.863837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077176.863837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077177.887853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077177.887934:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077178.911795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077178.911796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077179.039925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077179.039936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077179.039942:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077179.039958:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4167 00000100:00080000:2.0:1769077179.039974:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077179.039985:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077179.040033:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612681344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077179.040076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077179.040081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077179.040085:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077179.040092:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4167 00000100:00080000:2.0:1769077179.040096:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077179.040101:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077179.040112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612681344:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769077179.040115:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077179.040151:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612681472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077179.040165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077179.040169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077179.040171:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077179.040185:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4167 00000100:00080000:2.0:1769077179.040205:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077179.040211:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077179.040218:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077179.040240:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612681600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077179.040277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077179.040283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077179.040286:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077179.040294:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4167 00000100:00080000:2.0:1769077179.040312:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077179.040319:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077179.040336:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769077179.040346:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077179.040361:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612681728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077179.040553:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff8b1005d5aa00 x1855007612681344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077179.040583:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612681344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077179.040598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077179.040713:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077179.040725:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612681344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077179.040779:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681472:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077179.040829:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077179.040844:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff8b1005d5b100 x1855007612681472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077179.040886:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612681472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077179.040905:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077179.040910:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077179.040915:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077179.040925:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077179.040955:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077179.040970:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff8b1005d58380 x1855007612681600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077179.040981:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612681600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077179.040989:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077179.040994:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077179.040999:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077179.041008:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077179.041038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077179.041055:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077179/real 1769077179] req@ffff8b1005d5ad80 x1855007612681728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077179.041072:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612681728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077179.041090:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077179.041094:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077179.041099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077179.935825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077179.935886:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077180.959785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077180.959800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077181.983777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077181.983863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077183.007792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077183.007851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077184.031737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077184.031796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077184.159785:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077184.159794:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077184.159798:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077184.159806:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4172 00000100:00080000:3.0:1769077184.159816:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077184.159822:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077184.159862:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612681856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077184.159889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077184.159892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077184.159893:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077184.159896:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4172 00000100:00080000:3.0:1769077184.159899:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077184.159901:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077184.159909:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077184.159917:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612681984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077184.159924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077184.159927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077184.159928:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077184.159930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4172 00000100:00080000:3.0:1769077184.159932:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077184.159935:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077184.159938:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077184.159943:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612682112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077184.159950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077184.159953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077184.159954:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077184.159957:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4172 00000100:00080000:3.0:1769077184.159959:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077184.159961:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077184.159964:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077184.159971:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612682240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077184.159983:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612681856:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077184.160037:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077184.160065:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff8b10037dea00 x1855007612681856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.160079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612681856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.160087:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.160090:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077184.160093:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612681856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077184.160099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077184.160112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077184.160120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff8b10037ddc00 x1855007612681984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.160131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612681984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.160140:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.160143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077184.160147:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612681984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077184.160153:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077184.160173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077184.160185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff8b10037df100 x1855007612682112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.160198:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612682112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.160207:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.160211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077184.160214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077184.160236:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077184.160252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077184.160260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff8b10037dd880 x1855007612682240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.160269:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612682240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.160274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.160276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077184.160278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077185.055811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077185.055811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077186.079769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077186.079883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077187.103740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077187.103832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077188.127791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077188.127832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077189.151813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077189.151877:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077189.279797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077189.279810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077189.279814:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077189.279828:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4177 00000100:00080000:2.0:1769077189.279844:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077189.279853:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077189.279908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612682368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077189.279943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077189.279946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077189.279948:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077189.279951:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4177 00000100:00080000:2.0:1769077189.279955:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077189.279958:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077189.279968:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077189.279979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612682496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077189.279989:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077189.279993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077189.279995:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077189.280001:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612682368:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077189.280002:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4177 00000100:00080000:2.0:1769077189.280009:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077189.280015:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077189.280022:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077189.280037:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612682624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077189.280048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077189.280052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077189.280053:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077189.280057:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4177 00000100:00080000:2.0:1769077189.280060:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077189.280063:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077189.280068:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077189.280077:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612682752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077189.280088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077189.280130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff8b1005d59500 x1855007612682368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.280146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612682368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.280156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.280160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077189.280163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612682368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077189.280187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682496:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077189.280204:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077189.280230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff8b1005d59880 x1855007612682496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.280240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612682496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.280251:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.280253:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077189.280256:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077189.280260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077189.280275:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077189.280282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff8b1005d5b480 x1855007612682624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.280290:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612682624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.280295:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.280297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077189.280299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077189.280303:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077189.280314:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077189.280321:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff8b1005d58e00 x1855007612682752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.280329:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612682752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.280334:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.280336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077189.280339:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612682752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077190.175775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077190.175796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077191.199763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077191.199852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077192.223837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077192.223844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077193.247769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077193.247850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077194.271793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077194.271846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077194.399787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077194.399799:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077194.399805:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077194.399817:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4182 00000100:00080000:2.0:1769077194.399831:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077194.399841:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077194.399894:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612682880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077194.399921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077194.399924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077194.399926:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077194.399929:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4182 00000100:00080000:2.0:1769077194.399933:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077194.399936:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077194.399947:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077194.399959:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612683008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077194.399967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077194.399970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077194.399973:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077194.399977:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4182 00000100:00080000:2.0:1769077194.399980:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077194.399982:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077194.399986:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077194.399994:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612683136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077194.400010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077194.400014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612682880:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077194.400014:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077194.400017:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077194.400023:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4182 00000100:00080000:2.0:1769077194.400034:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077194.400038:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077194.400042:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077194.400053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612683264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077194.400095:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077194.400145:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff8b1005d5a300 x1855007612682880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.400161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612682880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.400169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.400173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077194.400176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612682880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077194.400183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077194.400202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077194.400212:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff8b1005d5bb80 x1855007612683008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.400220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612683008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.400226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.400229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077194.400231:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683008:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077194.400251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077194.400264:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077194.400271:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff8b1005d58000 x1855007612683136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.400279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612683136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.400284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.400287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077194.400288:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077194.400293:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077194.400303:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077194.400310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff8b1005d58a80 x1855007612683264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.400318:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612683264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.400323:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.400325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077194.400326:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077195.295797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077195.295837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077196.319710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077196.319771:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077197.343736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077197.343806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077198.367759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077198.367817:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077199.391836:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077199.391882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077199.519751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.519762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077199.519766:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.519776:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4187 00000100:00080000:3.0:1769077199.519788:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.519795:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077199.519833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612683392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.519873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.519878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077199.519880:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.519884:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4187 00000100:00080000:3.0:1769077199.519888:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.519893:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077199.519903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077199.519916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612683520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.519939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.519943:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077199.519945:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.519952:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4187 00000100:00100000:1.0:1769077199.519953:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612683392:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077199.519956:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.519965:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077199.519970:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077199.519983:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612683648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.519994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.519998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077199.520000:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.520004:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4187 00000100:00080000:3.0:1769077199.520009:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.520013:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769077199.520018:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769077199.520022:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077199.520063:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612683776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077199.520133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff8b10037dc000 x1855007612683392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.520153:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612683392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.520164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.520168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077199.520172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612683392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077199.520180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077199.520219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077199.520228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff8b10037dd500 x1855007612683520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.520237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612683520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.520243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.520245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077199.520247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077199.520266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077199.520279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077199.520286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff8b10037ddf80 x1855007612683648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.520294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612683648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.520300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.520302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077199.520304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077199.520308:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077199.520324:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077199.520336:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff8b10037de300 x1855007612683776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.520350:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612683776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.520359:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.520362:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077199.520366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612683776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077200.415749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077200.415772:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077201.439742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077201.439827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077202.463724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077202.463837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077203.487755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077203.487824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077204.511712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077204.511779:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077204.639736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077204.639745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077204.639750:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077204.639758:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4192 00000100:00080000:3.0:1769077204.639768:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077204.639773:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077204.639806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612683904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077204.639830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077204.639834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077204.639835:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077204.639838:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4192 00000100:00080000:3.0:1769077204.639840:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077204.639842:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077204.639849:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077204.639858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612684032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077204.639866:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077204.639869:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077204.639870:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077204.639873:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4192 00000100:00080000:3.0:1769077204.639875:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077204.639879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612683904:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077204.639880:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077204.639888:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077204.639895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612684160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077204.639902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077204.639904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077204.639906:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077204.639908:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4192 00000100:00080000:3.0:1769077204.639910:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077204.639913:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077204.639916:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077204.639923:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612684288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077204.639945:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077204.639992:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff8b10037ded80 x1855007612683904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.640006:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612683904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.640014:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.640017:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077204.640020:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612683904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077204.640026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684032:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077204.640042:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077204.640050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff8b10037de680 x1855007612684032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.640059:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612684032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.640065:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.640067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077204.640069:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077204.640087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077204.640099:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077204.640105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff8b10037dc380 x1855007612684160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.640113:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612684160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.640119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.640120:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077204.640122:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077204.640126:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077204.640136:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077204.640142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff8b10037dd180 x1855007612684288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.640149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612684288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.640159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.640161:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077204.640163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077205.535710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077205.535842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077206.559787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077206.559811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077207.583735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077207.583797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077208.607762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077208.607846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077209.631701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077209.631773:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077209.759742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.759751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077209.759755:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.759763:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4197 00000100:00080000:2.0:1769077209.759774:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.759779:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077209.759814:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612684416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.759841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.759846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077209.759848:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.759852:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4197 00000100:00080000:2.0:1769077209.759856:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.759873:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077209.759885:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077209.759898:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612684416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077209.759905:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612684544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.759916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.759919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077209.759921:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.759924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4197 00000100:00080000:2.0:1769077209.759928:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.759931:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077209.759935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077209.759946:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612684672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.759954:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.759957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077209.759958:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.759963:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4197 00000800:00000100:1.0:1769077209.759965:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769077209.759967:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.759972:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077209.759976:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077209.759986:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612684800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077209.760006:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff8b1005d59180 x1855007612684416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.760022:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612684416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.760031:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.760034:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077209.760037:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612684416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077209.760056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077209.760072:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077209.760081:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff8b1005d5b800 x1855007612684544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.760090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612684544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.760096:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.760098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077209.760101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077209.760106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077209.760118:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077209.760124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff8b1005d58a80 x1855007612684672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.760133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612684672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.760138:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.760140:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077209.760142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077209.760146:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077209.760158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077209.760168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff8b1005d58000 x1855007612684800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.760182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612684800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.760191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.760195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077209.760199:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612684800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077210.655747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077210.655802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077211.679772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077211.679882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077212.703803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077212.703902:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077213.727855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077213.727926:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077214.751760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077214.751841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077214.879847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.879859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077214.879865:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.879876:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4202 00000100:00080000:3.0:1769077214.879901:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.879913:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077214.879978:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612684928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077214.880039:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.880043:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077214.880045:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.880053:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4202 00000100:00080000:3.0:1769077214.880061:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.880067:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077214.880080:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077214.880098:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612685056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077214.880114:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612684928:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077214.880115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.880126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077214.880129:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.880140:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4202 00000100:00080000:3.0:1769077214.880150:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.880173:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077214.880184:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077214.880204:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612685184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077214.880217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.880221:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077214.880224:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.880228:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4202 00000100:00080000:3.0:1769077214.880245:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.880249:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769077214.880271:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769077214.880274:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077214.880307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff8b10037dca80 x1855007612684928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077214.880325:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612684928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077214.880335:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077214.880340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077214.880345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612684928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077214.880370:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077214.880376:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612685312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077214.880396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077214.880413:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff8b10037dd180 x1855007612685056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077214.880424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612685056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077214.880431:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077214.880434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077214.880436:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077214.880441:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077214.880455:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077214.880473:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff8b10037dc380 x1855007612685184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077214.880483:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612685184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077214.880489:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077214.880491:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077214.880493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077214.880503:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077214.880515:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077214.880522:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff8b10037de680 x1855007612685312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077214.880530:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612685312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077214.880536:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077214.880539:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077214.880542:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077215.775740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077215.775811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077216.799724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077216.799812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077217.823746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077217.823801:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077218.847778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077218.847883:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077219.871743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077219.872024:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077219.999821:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077219.999835:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077219.999841:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077219.999853:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4208 00000100:00080000:3.0:1769077219.999869:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077219.999877:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077219.999930:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612685440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077219.999972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077219.999978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077219.999981:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077219.999987:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4208 00000100:00080000:3.0:1769077219.999994:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077220.000002:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077220.000016:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077220.000033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612685440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077220.000038:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612685568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077220.000060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077220.000068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077220.000070:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077220.000076:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4208 00000100:00080000:3.0:1769077220.000080:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077220.000085:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077220.000091:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077220.000103:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612685696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077220.000113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077220.000121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769077220.000122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077220.000124:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077220.000134:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4208 00000100:00080000:3.0:1769077220.000139:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077220.000144:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077220.000150:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077220.000163:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612685824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077220.000173:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077220] req@ffff8b10037ded80 x1855007612685440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.000190:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612685440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077220.000200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077220.000208:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077220.000212:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612685440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077220.000233:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077220.000249:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077220.000258:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077220] req@ffff8b10037de300 x1855007612685568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.000266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612685568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077220.000272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077220.000274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077220.000276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077220.000280:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077220.000292:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077220.000298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077220] req@ffff8b10037ddf80 x1855007612685696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.000305:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612685696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077220.000315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077220.000317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077220.000319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077220.000323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077220.000334:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077220.000340:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077220] req@ffff8b10037dd500 x1855007612685824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077220.000347:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612685824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077220.000352:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077220.000354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077220.000356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612685824:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077220.895900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077220.895930:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077221.919748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077221.919853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077222.943797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077222.943883:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077223.967758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077223.967819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077224.991712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077224.991803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077225.119708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.119717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077225.119722:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.119742:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4213 00000100:00080000:3.0:1769077225.119754:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.119761:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077225.119796:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612685952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.119825:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.119829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077225.119831:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.119835:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4213 00000100:00080000:3.0:1769077225.119839:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.119842:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077225.119851:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077225.119863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612686080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077225.119885:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612685952:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077225.119886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.119895:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077225.119897:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.119901:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4213 00000100:00080000:3.0:1769077225.119904:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.119909:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077225.119915:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077225.119928:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612686208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.119949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.119953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077225.119955:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.119959:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4213 00000100:00080000:3.0:1769077225.119963:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077225.119969:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077225.119971:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077225.119978:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077225.119991:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612686336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077225.120017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff8b10037dc000 x1855007612685952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077225.120039:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612685952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077225.120048:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077225.120051:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077225.120054:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612685952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077225.120072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077225.120092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077225.120101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff8b10037dd880 x1855007612686080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077225.120111:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612686080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077225.120116:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077225.120119:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077225.120121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077225.120125:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077225.120137:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077225.120143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff8b10037df100 x1855007612686208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077225.120151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612686208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077225.120156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077225.120158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077225.120160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077225.120164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077225.120175:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077225.120181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff8b10037ddc00 x1855007612686336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077225.120189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612686336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077225.120194:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077225.120196:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077225.120198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077226.015721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077226.015778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077227.039762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077227.039830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077228.063780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077228.063809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077229.087768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077229.087817:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077230.111780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077230.111857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077230.239839:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.239852:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077230.239857:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.239869:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4218 00000100:00080000:2.0:1769077230.239882:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.239890:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077230.239942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612686464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.239973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.239976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077230.239978:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.239983:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4218 00000100:00080000:2.0:1769077230.239987:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.239991:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077230.240001:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077230.240009:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612686464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077230.240016:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612686592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.240026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.240029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077230.240031:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.240034:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4218 00000100:00080000:2.0:1769077230.240036:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.240038:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077230.240042:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077230.240052:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612686720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.240059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.240062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077230.240063:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.240066:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4218 00000100:00080000:2.0:1769077230.240068:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.240070:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077230.240073:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077230.240083:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612686848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077230.240135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077230.240183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077230.240214:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077231.135732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077231.135790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077232.159773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077232.159829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077233.183734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077233.183794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077233.311922:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077233.311968:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077233.311988:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077233.312040:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077233.312051:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077233.312060:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769077233.312105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077233] req@ffff8b1005d5a300 x1855007612686592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077233.312133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612686592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077233.312147:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077233.312153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077233.312157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077233.312165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077233] req@ffff8b1005d58e00 x1855007612686720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077233.312180:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612686720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077233.312189:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077233.312192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077233.312196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077233.312204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077233] req@ffff8b1005d5b480 x1855007612686848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077233.312216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612686848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077233.312245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077233.312249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077233.312253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612686848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077233.312261:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077233] req@ffff8b1005d5bb80 x1855007612686464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077233.312275:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612686464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077233.312284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077233.312288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077233.312291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612686464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077234.207741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077234.207791:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077235.231823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077235.231864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077235.359780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077235.359792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077235.359798:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077235.359809:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4223 00000100:00080000:2.0:1769077235.359822:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077235.359829:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077235.359873:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612686976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077235.359908:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077235.359911:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077235.359913:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077235.359916:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4223 00000100:00080000:2.0:1769077235.359919:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077235.359922:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077235.359931:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077235.359942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612687104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077235.359950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077235.359953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077235.359954:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077235.359957:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4223 00000100:00080000:2.0:1769077235.359959:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077235.359962:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077235.359967:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077235.359968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612686976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077235.359976:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612687232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077235.359983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077235.359985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077235.359986:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077235.359989:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4223 00000100:00080000:2.0:1769077235.359991:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077235.359994:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077235.359997:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077235.360005:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612687360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077235.360065:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077235.360117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff8b1005d59880 x1855007612686976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.360140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612686976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.360154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.360159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077235.360164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612686976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077235.360172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077235.360203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077235.360218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff8b1005d59500 x1855007612687104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.360233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612687104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.360242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.360246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077235.360250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077235.360280:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077235.360304:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077235.360315:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff8b1005d5ad80 x1855007612687232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.360327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612687232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.360335:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.360338:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077235.360342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077235.360348:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077235.360368:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077235.360380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff8b1005d58380 x1855007612687360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.360401:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612687360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.360410:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.360413:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077235.360417:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077236.255780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077236.255852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077237.279775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077237.279821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077238.303728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077238.303827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077239.327768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077239.327841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077240.351738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077240.351802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077240.479728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.479739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077240.479744:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.479755:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4228 00000100:00080000:2.0:1769077240.479766:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.479772:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077240.479804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612687488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077240.479834:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.479837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077240.479839:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.479842:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4228 00000100:00080000:2.0:1769077240.479844:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.479847:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077240.479854:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077240.479863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612687616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077240.479869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.479872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077240.479873:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.479875:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4228 00000100:00080000:2.0:1769077240.479877:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.479880:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077240.479883:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077240.479891:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612687744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077240.479904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.479907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077240.479909:0:8:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077240.479912:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612687488:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077240.479913:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4228 00000100:00080000:2.0:1769077240.479918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.479922:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077240.479925:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077240.479933:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612687872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077240.479973:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077240.480002:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff8b1005d5b100 x1855007612687488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.480015:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612687488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.480023:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.480026:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077240.480029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612687488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077240.480034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077240.480047:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077240.480054:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff8b1005d5aa00 x1855007612687616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.480062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612687616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.480072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.480074:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077240.480077:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077240.480094:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077240.480105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077240.480111:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff8b1005d58380 x1855007612687744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.480118:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612687744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.480124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.480125:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077240.480127:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077240.480131:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077240.480141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077240.480157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff8b1005d5ad80 x1855007612687872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.480165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612687872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.480170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.480172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077240.480175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612687872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077241.375745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077241.375808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077242.399766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077242.399815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077243.423742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077243.423801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077244.447772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077244.447838:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077245.471778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077245.471858:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077245.599780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077245.599789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077245.599793:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077245.599801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4233 00000100:00080000:3.0:1769077245.599810:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077245.599816:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077245.599848:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612688000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077245.599872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077245.599875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077245.599876:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077245.599879:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4233 00000100:00080000:3.0:1769077245.599881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077245.599885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077245.599892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077245.599899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612688128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077245.599907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077245.599909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077245.599911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077245.599913:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4233 00000100:00080000:3.0:1769077245.599915:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077245.599918:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077245.599922:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077245.599927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612688256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077245.599935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077245.599938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769077245.599940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612688000:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077245.599940:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077245.599945:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4233 00000100:00080000:3.0:1769077245.599949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077245.599953:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077245.599956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077245.599963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612688384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077245.600010:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077245.600045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077245] req@ffff8b10037dea00 x1855007612688000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077245.600064:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612688000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077245.600075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077245.600080:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077245.600085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612688000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077245.600093:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688128:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077245.600127:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077245.600143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077245] req@ffff8b10037dce00 x1855007612688128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077245.600157:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612688128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077245.600172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077245.600176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077245.600179:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077245.600203:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077245.600223:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077245.600235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077245] req@ffff8b10037df480 x1855007612688256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077245.600248:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612688256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077245.600257:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077245.600275:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077245.600279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077245.600286:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077245.600306:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077245.600319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077245] req@ffff8b10037df800 x1855007612688384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077245.600333:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612688384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077245.600342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077245.600345:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077245.600349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077246.495743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077246.495842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077247.519798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077247.519860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077248.543727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077248.543825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077249.567771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077249.567823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077250.591733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077250.591795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077250.719740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077250.719752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077250.719757:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077250.719769:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4238 00000100:00080000:3.0:1769077250.719779:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077250.719787:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077250.719824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612688512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077250.719859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077250.719864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077250.719865:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077250.719871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4238 00000100:00080000:3.0:1769077250.719876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077250.719883:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077250.719893:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077250.719918:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612688512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077250.719924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612688640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077250.719940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077250.719944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077250.719947:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077250.719952:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4238 00000800:00000100:0.0:1769077250.720004:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077250.720025:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077250.720034:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077250.720042:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077250.720059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612688768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077250.720075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077250.720078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077250.720081:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077250.720088:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4238 00000100:00100000:0.0:1769077250.720090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff8b10037dc700 x1855007612688512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077250.720095:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077250.720109:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077250.720115:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077250.720117:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612688512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077250.720133:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077250.720141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077250.720146:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612688512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077250.720171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077250.720212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077250.720223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff8b10037df800 x1855007612688640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077250.720234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612688640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077250.720242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077250.720245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077250.720247:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077250.720253:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077250.720274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077250.720278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612688896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077250.720288:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff8b10037df480 x1855007612688768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077250.720317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612688768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077250.720326:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077250.720332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077250.720336:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077250.720361:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077250.720376:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077250.720394:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff8b10037dce00 x1855007612688896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077250.720412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612688896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077250.720422:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077250.720425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077250.720427:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612688896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077251.615753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077251.615823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077252.639736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077252.639822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077253.663776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077253.663859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077254.687784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077254.687872:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077255.711707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077255.711752:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077255.839731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077255.839746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077255.839752:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077255.839766:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4243 00000100:00080000:3.0:1769077255.839780:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077255.839790:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077255.839837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612689024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077255.839873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077255.839877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077255.839880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077255.839886:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4243 00000100:00080000:3.0:1769077255.839891:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077255.839897:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077255.839910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077255.839911:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612689024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077255.839930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612689152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077255.839944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077255.839948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077255.839951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077255.839956:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4243 00000100:00080000:3.0:1769077255.839960:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077255.839965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077255.839970:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077255.839985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612689280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077255.839991:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077255.840003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077255.840007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077255.840010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077255.840016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4243 00000100:00080000:3.0:1769077255.840020:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077255.840025:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077255.840032:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff8b10037dea00 x1855007612689024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077255.840032:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077255.840050:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612689408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077255.840054:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612689024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077255.840078:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077255.840083:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077255.840109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612689024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077255.840134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689152:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077255.840153:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077255.840163:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff8b10037ddc00 x1855007612689152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077255.840172:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612689152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077255.840188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077255.840190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077255.840196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077255.840200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077255.840215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077255.840222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff8b10037df100 x1855007612689280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077255.840230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612689280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077255.840236:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077255.840238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077255.840240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077255.840243:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077255.840255:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077255.840260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff8b10037dd880 x1855007612689408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077255.840268:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612689408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077255.840274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077255.840277:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077255.840279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689408:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077256.735826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077256.735846:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077257.759808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077257.759883:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077258.783791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077258.783839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077259.807734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077259.807802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077260.831753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077260.831833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077260.959814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077260.959839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077260.959843:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077260.959851:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4249 00000100:00080000:3.0:1769077260.959862:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077260.959867:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077260.959917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612689536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077260.959944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077260.959948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077260.959951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077260.959971:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4249 00000100:00080000:3.0:1769077260.959975:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077260.959979:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077260.959988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077260.960016:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612689664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077260.960024:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612689536:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077260.960035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077260.960040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077260.960043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077260.960048:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4249 00000100:00080000:3.0:1769077260.960053:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077260.960057:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077260.960063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077260.960089:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612689792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077260.960097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077260.960100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769077260.960105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077260.960105:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077260.960112:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4249 00000100:00080000:3.0:1769077260.960117:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077260.960121:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077260.960125:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077260.960133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612689920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077260.960155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff8b10037dc000 x1855007612689536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077260.960173:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612689536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077260.960209:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077260.960213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077260.960216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612689536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077260.960235:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689664:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077260.960256:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077260.960269:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff8b10037dd500 x1855007612689664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077260.960283:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612689664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077260.960293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077260.960297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077260.960310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077260.960315:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077260.960333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077260.960341:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff8b10037ddf80 x1855007612689792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077260.960358:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612689792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077260.960364:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077260.960366:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077260.960368:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077260.960372:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077260.960383:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077260.960391:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff8b10037de300 x1855007612689920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077260.960403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612689920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077260.960422:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077260.960425:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077260.960429:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612689920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077261.855733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077261.855795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077262.879711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077262.879798:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077263.903740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077263.903817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077264.927749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077264.927802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077265.951721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077265.951824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077266.079794:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077266.079823:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077266.079828:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077266.079838:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4254 00000100:00080000:1.0:1769077266.079849:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077266.079856:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077266.079895:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612690048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077266.079951:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077266.079955:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077266.079957:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077266.079961:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4254 00000100:00080000:1.0:1769077266.079964:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077266.079968:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077266.079978:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077266.079992:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612690176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077266.080000:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612690048:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077266.080007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077266.080012:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077266.080014:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077266.080018:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4254 00000100:00080000:1.0:1769077266.080023:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077266.080026:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077266.080031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077266.080044:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612690304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077266.080053:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077266.080056:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077266.080057:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077266.080062:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4254 00000100:00080000:1.0:1769077266.080066:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077266.080069:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077266.080074:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077266.080076:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077266.080092:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612690432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077266.080121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff8b1006135c00 x1855007612690048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077266.080141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612690048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077266.080152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077266.080157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077266.080161:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612690048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077266.080182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690176:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077266.080203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077266.080213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff8b1006137b80 x1855007612690176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077266.080225:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612690176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077266.080253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077266.080257:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077266.080262:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690176:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077266.080270:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077266.080293:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077266.080304:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff8b1006135880 x1855007612690304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077266.080319:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612690304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077266.080347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077266.080350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077266.080353:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077266.080360:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077266.080380:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077266.080407:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff8b1006135500 x1855007612690432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077266.080418:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612690432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077266.080425:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077266.080427:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077266.080430:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077266.975745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077266.975809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077267.999699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077267.999813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077269.023824:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077269.023969:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077270.047734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077270.047961:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077271.071773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077271.071929:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077271.199818:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077271.199856:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077271.199877:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077271.199887:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4259 00000100:00080000:2.0:1769077271.199899:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077271.199907:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077271.199951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612690560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077271.199981:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077271.199985:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077271.199986:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077271.199990:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4259 00000100:00080000:2.0:1769077271.199993:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077271.199997:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077271.200006:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077271.200020:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612690688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077271.200029:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077271.200032:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077271.200033:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077271.200036:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4259 00000100:00080000:2.0:1769077271.200038:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077271.200041:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077271.200046:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077271.200056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612690816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077271.200073:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077271.200074:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612690560:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077271.200077:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077271.200086:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077271.200102:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4259 00000100:00080000:2.0:1769077271.200105:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077271.200108:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077271.200111:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077271.200125:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612690944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077271.200214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077271.200295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff8b1005d59500 x1855007612690560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077271.200326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612690560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077271.200340:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077271.200346:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077271.200351:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612690560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077271.200362:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690688:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077271.200393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077271.200408:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff8b1005d59880 x1855007612690688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077271.200421:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612690688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077271.200430:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077271.200433:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077271.200437:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077271.200491:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077271.200550:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077271.200573:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff8b1005d5bb80 x1855007612690816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077271.200588:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612690816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077271.200598:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077271.200602:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077271.200606:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077271.200614:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077271.200691:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077271.200706:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff8b1005d5b480 x1855007612690944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077271.200719:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612690944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077271.200728:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077271.200732:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077271.200735:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612690944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077272.095718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077272.095840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077273.119774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077273.119919:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077274.143739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077274.143961:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077275.167750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077275.167871:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077276.191688:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077276.191814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077276.319757:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077276.319769:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077276.319774:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077276.319785:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4264 00000100:00080000:2.0:1769077276.319797:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077276.319804:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077276.319835:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612691072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077276.319860:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077276.319865:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077276.319867:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077276.319871:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4264 00000100:00080000:2.0:1769077276.319875:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077276.319880:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077276.319891:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077276.319904:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612691200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077276.319912:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612691072:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077276.319921:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077276.319925:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077276.319926:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077276.319931:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4264 00000100:00080000:2.0:1769077276.319934:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077276.319937:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077276.319942:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077276.319954:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612691328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077276.319965:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077276.319968:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077276.319970:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077276.319974:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4264 00000100:00080000:2.0:1769077276.319978:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077276.319982:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769077276.319988:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769077276.319996:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077276.320009:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612691456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077276.320033:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff8b1005d58e00 x1855007612691072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077276.320055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612691072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077276.320063:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077276.320067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077276.320070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612691072:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077276.320090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691200:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077276.320114:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077276.320123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff8b1005d5a300 x1855007612691200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077276.320131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612691200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077276.320137:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077276.320140:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077276.320142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691200:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077276.320146:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077276.320158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077276.320164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff8b1005d58000 x1855007612691328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077276.320177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612691328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077276.320182:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077276.320184:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077276.320186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077276.320189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077276.320200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077276.320206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff8b1005d58a80 x1855007612691456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077276.320213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612691456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077276.320218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077276.320220:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077276.320222:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077277.215853:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077277.215979:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077278.239704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077278.239865:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077279.263759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077279.263836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077280.287797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077280.287890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077281.311819:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077281.311892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077281.439804:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077281.439836:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077281.439842:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077281.439851:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4269 00000100:00080000:2.0:1769077281.439863:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077281.439870:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077281.439908:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612691584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077281.439937:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077281.439941:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077281.439943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077281.439946:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4269 00000100:00080000:2.0:1769077281.439949:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077281.439953:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077281.439961:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077281.439971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612691712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077281.439980:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077281.439983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077281.439984:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077281.439987:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4269 00000100:00080000:2.0:1769077281.439990:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077281.439994:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077281.439998:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077281.440009:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612691840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077281.440034:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077281.440041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612691584:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077281.440041:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077281.440046:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077281.440055:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4269 00000100:00080000:2.0:1769077281.440059:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077281.440062:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077281.440066:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077281.440078:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612691968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077281.440129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077281.440166:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff8b1005d5b800 x1855007612691584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077281.440185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612691584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077281.440206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077281.440210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077281.440214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612691584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077281.440221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691712:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077281.440238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077281.440248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff8b1005d59180 x1855007612691712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077281.440260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612691712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077281.440267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077281.440269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077281.440273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691712:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077281.440300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077281.440328:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077281.440344:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff8b1005d58a80 x1855007612691840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077281.440362:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612691840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077281.440384:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077281.440389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077281.440394:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077281.440402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077281.440429:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077281.440444:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff8b1005d58000 x1855007612691968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077281.440463:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612691968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077281.440475:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077281.440480:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077281.440485:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612691968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077282.335793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077282.335858:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077283.359790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077283.359903:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077284.383723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077284.383793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077285.407776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077285.407804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077286.431734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077286.431786:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077286.559752:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.559760:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077286.559764:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.559773:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4274 00000100:00080000:2.0:1769077286.559782:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.559788:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077286.559820:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612692096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.559844:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.559847:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077286.559848:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.559851:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4274 00000100:00080000:2.0:1769077286.559854:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.559858:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077286.559866:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077286.559878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612692224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077286.559883:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612692096:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077286.559892:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.559896:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077286.559897:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.559901:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4274 00000100:00080000:2.0:1769077286.559905:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.559908:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077286.559912:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077286.559923:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612692352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.559931:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.559934:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077286.559936:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.559939:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4274 00000100:00080000:2.0:1769077286.559942:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077286.559946:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769077286.559948:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077286.559952:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077286.559961:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612692480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077286.559982:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff8b1005d5a300 x1855007612692096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077286.559996:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612692096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077286.560004:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077286.560007:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077286.560010:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612692096:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077286.560028:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692224:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077286.560042:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077286.560050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff8b1005d58e00 x1855007612692224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077286.560058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612692224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077286.560065:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077286.560068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077286.560070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692224:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077286.560074:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077286.560085:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077286.560091:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff8b1005d5b480 x1855007612692352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077286.560099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612692352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077286.560104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077286.560106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077286.560108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077286.560112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077286.560123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077286.560129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff8b1005d5bb80 x1855007612692480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077286.560137:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612692480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077286.560142:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077286.560143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077286.560145:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077287.455744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077287.455820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077288.479817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077288.479908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077289.503755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077289.503914:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077290.527755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077290.527823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077291.551778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077291.551863:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077291.679848:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077291.679864:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077291.679869:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077291.679881:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4279 00000100:00080000:1.0:1769077291.679895:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077291.679901:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077291.679951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612692608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077291.679995:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077291.679999:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077291.680001:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077291.680005:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4279 00000100:00080000:1.0:1769077291.680008:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077291.680012:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077291.680022:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077291.680035:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612692736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077291.680043:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077291.680047:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077291.680048:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077291.680051:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4279 00000100:00080000:1.0:1769077291.680054:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077291.680057:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077291.680060:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077291.680067:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612692864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077291.680078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612692608:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077291.680080:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077291.680092:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077291.680097:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077291.680105:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4279 00000100:00080000:1.0:1769077291.680109:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077291.680116:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077291.680123:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077291.680138:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612692992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077291.680177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077291.680218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff8b1006136d80 x1855007612692608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077291.680239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612692608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077291.680253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077291.680258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077291.680263:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612692608:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077291.680283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692736:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077291.680308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077291.680322:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff8b1006136680 x1855007612692736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077291.680335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612692736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077291.680345:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077291.680348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077291.680352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692736:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077291.680389:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077291.680413:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077291.680427:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff8b1006137100 x1855007612692864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077291.680448:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612692864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077291.680458:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077291.680461:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077291.680465:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077291.680472:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077291.680510:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077291.680523:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff8b1006134700 x1855007612692992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077291.680536:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612692992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077291.680545:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077291.680549:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077291.680553:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612692992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077292.575790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077292.575887:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077293.599725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077293.599796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077294.623793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077294.623842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077295.647733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077295.647820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077296.671758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077296.671801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077296.799718:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077296.799727:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077296.799733:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077296.799743:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4284 00000100:00080000:1.0:1769077296.799753:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077296.799760:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077296.799797:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612693120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077296.799825:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077296.799829:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077296.799831:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077296.799835:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4284 00000100:00080000:1.0:1769077296.799838:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077296.799842:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077296.799851:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077296.799863:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612693248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077296.799875:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612693120:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077296.799875:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077296.799885:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077296.799886:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077296.799892:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4284 00000100:00080000:1.0:1769077296.799895:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077296.799899:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077296.799903:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077296.799915:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612693376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077296.799924:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077296.799927:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077296.799928:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077296.799931:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4284 00000100:00080000:1.0:1769077296.799937:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077296.799940:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077296.799944:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077296.799954:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612693504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077296.799987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077296.800015:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077296.800032:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077297.695707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077297.695787:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077298.719759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077298.719827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077299.743715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077299.743793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077299.871948:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769077299.871986:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077299.872001:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077299.872033:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077299.872040:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077299.872046:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769077299.872098:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077299] req@ffff8b1006134a80 x1855007612693248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077299.872131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612693248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077299.872143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077299.872149:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077299.872153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077299.872162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077299] req@ffff8b1006137480 x1855007612693376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077299.872175:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612693376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077299.872184:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077299.872187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077299.872191:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077299.872198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077299] req@ffff8b1006134700 x1855007612693504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077299.872215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612693504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077299.872223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077299.872226:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077299.872230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077299.872237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077299] req@ffff8b1006134e00 x1855007612693120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077299.872250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612693120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077299.872258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077299.872261:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077299.872265:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612693120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077300.767787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077300.767844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077301.791712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077301.791778:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077301.919773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077301.919781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077301.919784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077301.919793:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4290 00000100:00080000:3.0:1769077301.919802:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077301.919808:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077301.919838:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612693632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077301.919864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077301.919867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077301.919868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077301.919871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4290 00000100:00080000:3.0:1769077301.919874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077301.919876:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077301.919883:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077301.919891:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612693760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077301.919898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077301.919901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077301.919902:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077301.919905:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4290 00000100:00080000:3.0:1769077301.919907:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077301.919909:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077301.919913:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077301.919923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612693888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077301.919932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612693632:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077301.919936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077301.919943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077301.919944:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077301.919950:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4290 00000100:00080000:3.0:1769077301.919954:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077301.919958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077301.919964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077301.919976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612694016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077301.920008:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077301.920046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff8b10037ded80 x1855007612693632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077301.920066:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612693632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077301.920080:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077301.920086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077301.920091:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612693632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077301.920100:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693760:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077301.920126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077301.920141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff8b10037de680 x1855007612693760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077301.920157:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612693760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077301.920166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077301.920170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077301.920174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693760:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077301.920205:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077301.920230:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077301.920245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff8b10037dc380 x1855007612693888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077301.920258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612693888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077301.920268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077301.920271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077301.920275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612693888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077301.920281:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077301.920304:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077301.920359:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff8b10037dd180 x1855007612694016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077301.920374:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612694016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077301.920384:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077301.920388:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077301.920399:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077302.815795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077302.815863:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077303.839783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077303.839838:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077304.863768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077304.863850:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077305.887738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077305.887826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077306.911739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077306.911949:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077307.039788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.039797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077307.039800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.039808:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4295 00000100:00080000:0.0:1769077307.039818:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.039824:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077307.039862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612694144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.039888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.039892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.039894:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.039898:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4295 00000100:00080000:0.0:1769077307.039902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.039916:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077307.039925:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077307.039941:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612694272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.039952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077307.039955:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612694144:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077307.039956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077307.039960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.039968:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4295 00000100:00080000:0.0:1769077307.039973:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077307.039978:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077307.039985:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077307.039999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612694400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077307.040010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077307.040014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077307.040015:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077307.040018:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4295 00000100:00080000:0.0:1769077307.040023:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077307.040026:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077307.040031:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077307.040038:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077307.040051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612694528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077307.040079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff8b1006024a80 x1855007612694144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077307.040100:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612694144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077307.040112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077307.040117:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077307.040121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612694144:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077307.040160:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694272:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077307.040184:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077307.040198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff8b1006025f80 x1855007612694272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077307.040212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612694272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077307.040221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077307.040256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077307.040261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694272:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077307.040268:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077307.040290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077307.040302:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff8b1006027b80 x1855007612694400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077307.040316:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612694400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077307.040332:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077307.040336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077307.040339:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077307.040346:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077307.040365:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077307.040376:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077307/real 1769077307] req@ffff8b1006026d80 x1855007612694528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077307.040388:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612694528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077307.040397:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077307.040400:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077307.040403:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077307.935775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077307.935867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077308.959736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077308.959802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077309.983810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077309.983861:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077311.007751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077311.007857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077312.031800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077312.031832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077312.159814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.159827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077312.159832:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.159842:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4300 00000100:00080000:3.0:1769077312.159855:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.159862:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077312.159906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612694656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.159945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.159950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077312.159953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.159959:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4300 00000100:00080000:3.0:1769077312.159964:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.159970:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077312.159983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077312.159995:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612694656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077312.160001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612694784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.160011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.160014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077312.160015:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.160019:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4300 00000100:00080000:3.0:1769077312.160022:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.160025:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077312.160029:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077312.160039:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612694912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.160046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.160048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077312.160050:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.160052:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4300 00000100:00080000:3.0:1769077312.160055:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.160057:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077312.160060:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077312.160068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612695040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077312.160075:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077312.160118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077312] req@ffff8b10037dca80 x1855007612694656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077312.160135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612694656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077312.160164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077312.160168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077312.160172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612694656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077312.160197:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694784:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077312.160216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077312.160226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077312] req@ffff8b10037dd180 x1855007612694784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077312.160237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612694784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077312.160246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077312.160250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077312.160254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077312.160260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077312.160282:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077312.160295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077312] req@ffff8b10037dc380 x1855007612694912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077312.160308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612694912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077312.160318:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077312.160322:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077312.160325:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612694912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077312.160332:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077312.160349:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077312.160357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077312] req@ffff8b10037de680 x1855007612695040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077312.160366:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612695040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077312.160372:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077312.160374:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077312.160376:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077313.055709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077313.055797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077314.079730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077314.079812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077315.103720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077315.103762:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077316.127743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077316.127797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077317.151719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077317.151749:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077317.279751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.279759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077317.279764:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.279774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4305 00000100:00080000:3.0:1769077317.279785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.279792:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077317.279826:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612695168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.279853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.279857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077317.279859:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.279863:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4305 00000100:00080000:3.0:1769077317.279867:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.279871:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077317.279879:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077317.279897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612695296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.279909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077317.279910:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612695168:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077317.279917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077317.279920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.279926:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4305 00000100:00080000:3.0:1769077317.279930:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.279935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077317.279940:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077317.279953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612695424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.279963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.279966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077317.279968:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.279972:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4305 00000800:00000100:1.0:1769077317.279977:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077317.279978:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.279984:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077317.279990:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077317.280001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612695552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077317.280015:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff8b10037ded80 x1855007612695168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.280034:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612695168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.280081:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.280086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077317.280090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612695168:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077317.280110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695296:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077317.280131:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077317.280142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff8b10037de300 x1855007612695296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.280154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612695296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.280167:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.280170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077317.280173:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695296:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077317.280178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077317.280195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077317.280204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff8b10037ddf80 x1855007612695424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.280214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612695424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.280221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.280224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077317.280227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077317.280232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077317.280247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077317.280255:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff8b10037dd500 x1855007612695552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.280265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612695552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.280272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.280275:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077317.280278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077318.175743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077318.175801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077319.199700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077319.199752:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077320.223739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077320.223801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077321.247724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077321.247754:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077322.271735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077322.271797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077322.399730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077322.399742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077322.399747:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077322.399758:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4310 00000100:00080000:3.0:1769077322.399770:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077322.399777:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077322.399817:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612695680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077322.399842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077322.399844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077322.399846:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077322.399849:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4310 00000100:00080000:3.0:1769077322.399851:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077322.399853:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077322.399861:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077322.399869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612695808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077322.399876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077322.399878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077322.399879:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077322.399882:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4310 00000100:00080000:3.0:1769077322.399884:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077322.399886:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077322.399890:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077322.399897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612695936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077322.399909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077322.399912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077322.399914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077322.399919:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4310 00000100:00080000:3.0:1769077322.399923:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077322.399927:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077322.399933:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077322.399938:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612695680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077322.399949:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612696064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077322.400021:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077322.400062:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff8b10037dc000 x1855007612695680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.400090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612695680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.400101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.400107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077322.400111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612695680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077322.400121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695808:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077322.400142:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077322.400159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff8b10037dd880 x1855007612695808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.400174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612695808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.400187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.400193:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077322.400199:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077322.400208:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077322.400232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077322.400252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff8b10037df100 x1855007612695936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.400265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612695936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.400273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.400276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077322.400279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612695936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077322.400302:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077322.400320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077322.400339:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff8b10037ddc00 x1855007612696064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.400351:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612696064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.400359:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.400363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077322.400366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077323.295731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077323.295778:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077324.319742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077324.319789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077325.343742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077325.343833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077326.367764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077326.367842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077327.391685:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077327.391798:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077327.519776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077327.519784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077327.519788:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077327.519795:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4315 00000100:00080000:2.0:1769077327.519804:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077327.519810:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077327.519841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612696192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077327.519864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077327.519867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077327.519868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077327.519870:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4315 00000100:00080000:2.0:1769077327.519872:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077327.519875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077327.519895:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077327.519907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612696320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077327.519915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077327.519917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077327.519919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077327.519921:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4315 00000100:00080000:2.0:1769077327.519924:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077327.519927:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077327.519932:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077327.519940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612696192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077327.519940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612696448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077327.519963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077327.519965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077327.519967:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077327.519970:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4315 00000100:00080000:2.0:1769077327.519973:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077327.519975:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077327.519980:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077327.519987:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612696576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077327.520038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077327.520087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff8b1005d59880 x1855007612696192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.520105:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612696192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.520114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.520117:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077327.520121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612696192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077327.520127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696320:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077327.520145:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077327.520153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff8b1005d59500 x1855007612696320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.520161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612696320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.520166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.520168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077327.520171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077327.520194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077327.520205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077327.520211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff8b1005d5ad80 x1855007612696448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.520217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612696448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.520223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.520224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077327.520226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077327.520230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077327.520240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077327.520245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff8b1005d58380 x1855007612696576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.520252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612696576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.520257:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.520259:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077327.520260:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077328.415729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077328.415824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077329.439736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077329.439830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077330.463737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077330.463807:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077331.487773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077331.487865:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077332.511724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077332.511831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077332.639794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077332.639836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077332.639842:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077332.639855:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4320 00000100:00080000:1.0:1769077332.639869:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077332.639876:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077332.639923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612696704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077332.639956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077332.639959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077332.639961:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077332.639964:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4320 00000100:00080000:1.0:1769077332.639966:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077332.639970:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077332.639979:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077332.639989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612696832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077332.639997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077332.640001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077332.640002:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077332.640005:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4320 00000100:00080000:1.0:1769077332.640007:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077332.640009:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077332.640014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077332.640022:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612696960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077332.640035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612696704:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077332.640037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077332.640054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077332.640055:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077332.640059:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4320 00000100:00080000:1.0:1769077332.640061:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077332.640063:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077332.640068:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077332.640079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612697088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077332.640118:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077332.640150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff8b1006137100 x1855007612696704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077332.640166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612696704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077332.640175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077332.640178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077332.640181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612696704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077332.640189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696832:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077332.640202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077332.640210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff8b1006136680 x1855007612696832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077332.640218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612696832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077332.640223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077332.640225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077332.640227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696832:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077332.640247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077332.640259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077332.640265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff8b1006136d80 x1855007612696960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077332.640273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612696960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077332.640278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077332.640280:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077332.640282:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612696960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077332.640286:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077332.640296:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077332.640302:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff8b1006135500 x1855007612697088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077332.640310:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612697088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077332.640315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077332.640317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077332.640319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077333.535732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077333.535792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077334.559711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077334.559828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077335.583743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077335.583825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077336.607704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077336.607787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077337.631771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077337.631834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077337.759702:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077337.759720:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077337.759724:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077337.759731:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4325 00000100:00080000:0.0:1769077337.759741:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077337.759746:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077337.759776:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612697216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077337.759798:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077337.759801:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077337.759802:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077337.759805:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4325 00000100:00080000:0.0:1769077337.759808:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077337.759812:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077337.759820:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077337.759831:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612697344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077337.759832:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612697216:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077337.759846:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077337.759848:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077337.759850:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077337.759852:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4325 00000100:00080000:0.0:1769077337.759854:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077337.759857:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077337.759860:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077337.759870:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612697472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077337.759879:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077337.759882:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077337.759884:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077337.759888:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4325 00000100:00080000:0.0:1769077337.759892:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077337.759894:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077337.759899:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077337.759907:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077337.759921:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612697600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077337.759925:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff8b1006026a00 x1855007612697216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.759959:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612697216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.759967:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.759971:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077337.759974:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612697216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077337.759991:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697344:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077337.760007:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077337.760023:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff8b1006024000 x1855007612697344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.760032:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612697344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.760037:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.760040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077337.760042:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077337.760046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077337.760059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077337.760065:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff8b1006025880 x1855007612697472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.760078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612697472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.760083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.760084:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077337.760087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077337.760091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077337.760109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077337.760120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff8b1006026300 x1855007612697600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.760133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612697600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.760142:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.760146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077337.760149:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077338.655785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077338.655836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077339.679768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077339.679824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077340.703763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077340.703836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077341.727743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077341.727815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077342.751784:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077342.751794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077342.879777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077342.879787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077342.879792:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077342.879802:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4330 00000100:00080000:2.0:1769077342.879813:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077342.879820:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077342.879856:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612697728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077342.879885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077342.879889:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077342.879891:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077342.879896:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4330 00000100:00080000:2.0:1769077342.879899:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077342.879904:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077342.879913:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077342.879925:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612697856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077342.879935:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077342.879939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077342.879941:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077342.879946:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4330 00000100:00080000:2.0:1769077342.879950:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077342.879954:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077342.879960:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077342.879968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612697728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077342.879975:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612697984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077342.879988:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077342.879992:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077342.879994:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077342.879999:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4330 00000100:00080000:2.0:1769077342.880003:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077342.880007:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077342.880012:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077342.880024:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612698112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077342.880029:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077342.880064:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff8b1005d5aa00 x1855007612697728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.880079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612697728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.880091:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.880094:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077342.880097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612697728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077342.880103:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697856:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077342.880116:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077342.880124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff8b1005d5b100 x1855007612697856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.880132:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612697856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.880137:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.880139:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077342.880141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697856:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077342.880159:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077342.880170:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077342.880177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff8b1005d58380 x1855007612697984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.880184:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612697984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.880189:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.880191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077342.880193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612697984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077342.880197:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077342.880207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077342.880213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff8b1005d5ad80 x1855007612698112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.880220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612698112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.880226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.880227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077342.880229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077343.775822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077343.775946:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077344.799765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077344.799801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077345.823763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077345.823849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077346.847717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077346.847792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077347.871741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077347.871840:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077347.999742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077347.999751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077347.999754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077347.999763:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4336 00000100:00080000:2.0:1769077347.999773:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077347.999778:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077347.999809:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612698240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077347.999837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077347.999840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077347.999841:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077347.999844:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4336 00000100:00080000:2.0:1769077347.999847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077347.999849:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077347.999856:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077347.999864:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612698368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077347.999870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077347.999873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077347.999874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077347.999877:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4336 00000100:00080000:2.0:1769077347.999879:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077347.999882:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077347.999885:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077347.999898:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612698496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077347.999907:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612698240:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077347.999910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077347.999917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077347.999918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077347.999922:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4336 00000100:00080000:2.0:1769077347.999924:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077347.999927:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077347.999930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077347.999938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612698624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077347.999980:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077348.000019:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff8b1005d59500 x1855007612698240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077348.000039:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612698240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077348.000051:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077348.000056:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077348.000060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612698240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077348.000069:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698368:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077348.000093:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077348.000107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077348] req@ffff8b1005d59880 x1855007612698368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077348.000123:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612698368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077348.000132:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077348.000136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077348.000140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698368:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077348.000165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077348.000188:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077348.000201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077348] req@ffff8b1005d5bb80 x1855007612698496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077348.000214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612698496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077348.000223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077348.000227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077348.000230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077348.000237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077348.000258:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077348.000270:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077348] req@ffff8b1005d5b480 x1855007612698624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077348.000283:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612698624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077348.000299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077348.000303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077348.000307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077348.895760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077348.895802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077349.919818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077349.919834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077350.943755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077350.943805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077351.967739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077351.967800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077352.991733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077352.991794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077353.119717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077353.119726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077353.119731:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077353.119739:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4341 00000100:00080000:2.0:1769077353.119749:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077353.119755:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077353.119787:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612698752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077353.119807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077353.119810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077353.119811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077353.119814:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4341 00000100:00080000:2.0:1769077353.119816:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077353.119818:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077353.119825:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077353.119834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612698880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077353.119842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077353.119845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077353.119846:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077353.119850:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4341 00000100:00100000:1.0:1769077353.119851:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612698752:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077353.119852:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077353.119858:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077353.119863:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077353.119871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612699008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077353.119877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077353.119879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077353.119881:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077353.119883:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4341 00000100:00080000:2.0:1769077353.119885:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077353.119888:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077353.119891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077353.119897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612699136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077353.119911:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077353.119941:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff8b1005d58e00 x1855007612698752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.119955:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612698752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.119962:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.119966:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077353.119969:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612698752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077353.119988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698880:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077353.120003:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077353.120011:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff8b1005d5a300 x1855007612698880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.120019:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612698880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.120025:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.120027:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077353.120029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612698880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077353.120034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077353.120055:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077353.120066:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff8b1005d58000 x1855007612699008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.120074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612699008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.120080:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.120082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077353.120084:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077353.120088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077353.120099:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077353.120105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff8b1005d58a80 x1855007612699136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.120112:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612699136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.120117:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.120119:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077353.120121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077354.015709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077354.015761:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077355.039793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077355.039851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077356.063726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077356.063767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077357.087837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077357.087882:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077358.111775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077358.111803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077358.239850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.239864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077358.239868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.239877:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4346 00000100:00080000:3.0:1769077358.239888:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.239895:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077358.239927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612699264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.239953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.239956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077358.239957:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.239960:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4346 00000100:00080000:3.0:1769077358.239964:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.239967:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077358.239975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077358.239991:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612699392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.240001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.240003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077358.240005:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.240009:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4346 00000100:00080000:3.0:1769077358.240013:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.240017:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077358.240021:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077358.240029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612699264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077358.240037:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612699520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.240047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.240050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077358.240051:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.240058:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4346 00000100:00080000:3.0:1769077358.240061:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.240064:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077358.240067:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077358.240079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612699648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077358.240119:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077358.240156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff8b10037dea00 x1855007612699264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077358.240176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612699264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077358.240187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077358.240194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077358.240199:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612699264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077358.240209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699392:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077358.240234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077358.240254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff8b10037dce00 x1855007612699392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077358.240263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612699392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077358.240270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077358.240272:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077358.240275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699392:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077358.240304:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077358.240332:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077358.240342:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff8b10037df480 x1855007612699520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077358.240353:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612699520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077358.240360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077358.240363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077358.240384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077358.240390:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077358.240416:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077358.240429:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff8b10037df800 x1855007612699648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077358.240446:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612699648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077358.240452:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077358.240455:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077358.240458:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077359.135812:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077359.135874:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077360.159761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077360.159817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077361.183734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077361.183793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077362.207745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077362.207832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077363.231783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077363.231852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077363.359750:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.359760:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077363.359763:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.359771:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4351 00000100:00080000:3.0:1769077363.359781:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.359786:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077363.359818:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612699776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.359841:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.359844:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077363.359846:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.359848:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4351 00000100:00080000:3.0:1769077363.359851:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.359855:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077363.359863:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077363.359878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612699904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077363.359879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612699776:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077363.359895:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.359900:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077363.359902:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.359907:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4351 00000100:00080000:3.0:1769077363.359930:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.359936:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077363.359943:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077363.359963:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612700032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.359977:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077363.359978:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699904:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769077363.359985:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077363.359991:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.359997:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4351 00000100:00080000:3.0:1769077363.360002:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.360007:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077363.360015:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077363.360023:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077363.360032:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612700160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077363.360057:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077364.255832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077364.255892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077365.279727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077365.279814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077366.303717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077366.303825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077366.431961:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:0.0:1769077366.432002:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077366.432046:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077366.432086:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077366.432112:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077366.432123:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769077366.432174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077366] req@ffff8b10037dce00 x1855007612700160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077366.432204:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612700160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077366.432219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077366.432224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077366.432242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077366.432251:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077366] req@ffff8b10037df480 x1855007612700032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077366.432266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612700032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077366.432276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077366.432280:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077366.432283:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077366.432290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077366] req@ffff8b10037df800 x1855007612699904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077366.432309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612699904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077366.432317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077366.432320:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077366.432324:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612699904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077366.432330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077366] req@ffff8b10037dc700 x1855007612699776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077366.432343:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612699776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077366.432351:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077366.432355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077366.432358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612699776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077367.327744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077367.327804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077368.351751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077368.351851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077368.479735:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.479748:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077368.479753:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.479764:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4356 00000100:00080000:3.0:1769077368.479777:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.479785:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077368.479825:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612700288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.479862:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.479867:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077368.479869:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.479874:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4356 00000100:00080000:3.0:1769077368.479878:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.479884:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077368.479898:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077368.479915:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612700416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.479930:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.479939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077368.479941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612700288:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077368.479943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.479959:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4356 00000100:00080000:3.0:1769077368.479964:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.479970:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077368.479986:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077368.480012:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612700544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.480026:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077368.480030:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077368.480038:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077368.480041:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.480050:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4356 00000100:00080000:3.0:1769077368.480058:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.480065:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077368.480074:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077368.480086:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff8b10037dea00 x1855007612700288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077368.480095:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612700672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077368.480144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612700288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077368.480160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077368.480168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077368.480172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612700288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077368.480232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700416:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077368.480263:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077368.480279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff8b10037ddc00 x1855007612700416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077368.480293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612700416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077368.480300:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077368.480303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077368.480307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700416:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077368.480316:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077368.480341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077368.480354:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff8b10037df100 x1855007612700544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077368.480367:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612700544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077368.480375:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077368.480379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077368.480381:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077368.480387:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077368.480409:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077368.480421:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff8b10037dd880 x1855007612700672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077368.480434:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612700672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077368.480442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077368.480445:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077368.480448:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077369.375700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077369.375791:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077370.399770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077370.399785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077371.423757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077371.423773:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077372.447764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077372.447842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077373.471730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077373.471812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077373.599842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.599859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077373.599865:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.599879:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4361 00000100:00080000:0.0:1769077373.599893:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.599900:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077373.599975:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612700800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.600003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.600007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077373.600009:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.600012:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4361 00000100:00080000:0.0:1769077373.600016:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.600019:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077373.600028:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077373.600060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612700928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.600069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.600072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077373.600075:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.600079:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4361 00000100:00080000:0.0:1769077373.600081:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.600086:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077373.600091:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077373.600103:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612701056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077373.600149:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612700800:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077373.600152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.600157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077373.600158:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.600162:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4361 00000100:00080000:0.0:1769077373.600165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.600169:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077373.600174:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077373.600202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612701184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077373.600249:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077373.600310:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff8b1006025180 x1855007612700800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077373.600331:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612700800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077373.600360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077373.600365:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077373.600370:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612700800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077373.600379:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700928:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077373.600406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077373.600423:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff8b1006027800 x1855007612700928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077373.600437:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612700928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077373.600446:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077373.600450:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077373.600454:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612700928:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077373.600484:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077373.600504:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077373.600512:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff8b1006027100 x1855007612701056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077373.600521:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612701056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077373.600527:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077373.600530:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077373.600532:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077373.600535:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077373.600547:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077373.600553:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff8b1006026680 x1855007612701184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077373.600566:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612701184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077373.600572:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077373.600574:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077373.600576:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077374.495748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077374.495839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077375.519705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077375.519758:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077376.543746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077376.543812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077377.567777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077377.567846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077378.591810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077378.591829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077378.719742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077378.719754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077378.719759:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077378.719775:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4366 00000100:00080000:3.0:1769077378.719786:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077378.719794:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077378.719835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612701312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077378.719866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077378.719870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077378.719872:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077378.719877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4366 00000100:00080000:3.0:1769077378.719881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077378.719885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077378.719896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077378.719912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612701440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077378.719924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077378.719929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077378.719933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077378.719934:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612701312:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077378.719941:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4366 00000100:00080000:3.0:1769077378.719947:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077378.719950:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077378.719955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077378.719970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612701568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077378.719981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077378.719985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077378.719987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077378.719992:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4366 00000100:00080000:3.0:1769077378.719997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077378.720001:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077378.720007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077378.720014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077378.720054:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077378] req@ffff8b10037dc000 x1855007612701312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077378.720058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612701696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077378.720080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612701312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077378.720089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077378.720093:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077378.720097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612701312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077378.720117:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701440:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077378.720135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077378.720144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077378] req@ffff8b10037dd500 x1855007612701440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077378.720153:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612701440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077378.720159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077378.720161:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077378.720163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701440:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077378.720167:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077378.720180:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077378.720190:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077378] req@ffff8b10037ddf80 x1855007612701568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077378.720204:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612701568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077378.720212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077378.720216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077378.720220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077378.720227:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077378.720247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077378.720260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077378] req@ffff8b10037de300 x1855007612701696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077378.720273:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612701696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077378.720288:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077378.720292:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077378.720296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077379.615740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077379.615826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077380.639797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077380.639845:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077381.663778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077381.663822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077382.687764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077382.687848:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077383.711753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077383.711816:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077383.839790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077383.839799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077383.839803:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077383.839811:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4371 00000100:00080000:0.0:1769077383.839820:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077383.839826:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077383.839858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612701824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077383.839889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077383.839893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077383.839895:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077383.839900:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4371 00000100:00080000:0.0:1769077383.839904:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077383.839907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077383.839914:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077383.839928:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612701952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077383.839953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077383.839958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077383.839962:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077383.839963:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612701824:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077383.839970:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4371 00000100:00080000:0.0:1769077383.839975:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077383.839980:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077383.839986:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077383.840001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612702080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077383.840012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077383.840016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077383.840018:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077383.840025:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077383.840026:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4371 00000100:00080000:0.0:1769077383.840033:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077383.840039:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077383.840058:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077383.840059:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff8b1006026680 x1855007612701824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077383.840076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612702208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077383.840079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612701824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077383.840108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077383.840114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077383.840118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612701824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077383.840137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701952:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077383.840153:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077383.840161:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff8b1006027100 x1855007612701952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077383.840170:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612701952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077383.840176:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077383.840187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077383.840189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612701952:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077383.840194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077383.840207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077383.840213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff8b1006027800 x1855007612702080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077383.840221:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612702080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077383.840227:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077383.840229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077383.840232:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077383.840239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077383.840259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077383.840270:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff8b1006025180 x1855007612702208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077383.840284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612702208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077383.840293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077383.840296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077383.840300:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077384.735732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077384.735792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077385.759769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077385.759780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077386.783746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077386.783834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077387.807741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077387.807805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077388.831732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077388.831798:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077388.959778:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077388.959787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077388.959791:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077388.959798:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4377 00000100:00080000:2.0:1769077388.959809:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077388.959816:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077388.959860:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612702336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077388.959886:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077388.959889:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077388.959891:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077388.959894:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4377 00000100:00080000:2.0:1769077388.959896:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077388.959899:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077388.959907:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077388.959918:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612702464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077388.959925:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077388.959928:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077388.959930:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077388.959934:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4377 00000100:00080000:2.0:1769077388.959937:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077388.959938:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612702336:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077388.959941:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077388.959950:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077388.959960:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612702592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077388.959967:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077388.959970:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077388.959971:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077388.959973:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4377 00000100:00080000:2.0:1769077388.959976:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077388.959978:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077388.959982:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077388.959989:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612702720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077388.960001:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077388.960031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff8b1005d59180 x1855007612702336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077388.960062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612702336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077388.960072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077388.960075:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077388.960079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612702336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077388.960085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702464:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077388.960111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077388.960135:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff8b1005d5b800 x1855007612702464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077388.960144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612702464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077388.960150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077388.960152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077388.960154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077388.960185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077388.960198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077388.960205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff8b1005d58a80 x1855007612702592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077388.960213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612702592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077388.960223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077388.960225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077388.960227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077388.960231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077388.960241:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077388.960247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff8b1005d58000 x1855007612702720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077388.960255:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612702720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077388.960277:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077388.960279:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077388.960281:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077389.855763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077389.855825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077390.879740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077390.879833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077391.903750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077391.903867:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077392.927743:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077392.927761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077393.951804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077393.951862:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077394.079783:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077394.079793:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077394.079799:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077394.079810:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4382 00000100:00080000:2.0:1769077394.079824:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077394.079834:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077394.079877:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612702848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077394.079904:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077394.079909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077394.079911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077394.079916:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4382 00000100:00080000:2.0:1769077394.079919:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077394.079923:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077394.079936:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077394.079956:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612702976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077394.079977:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077394.079985:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769077394.079988:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612702848:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077394.079990:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077394.080002:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4382 00000100:00080000:2.0:1769077394.080008:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077394.080014:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077394.080022:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077394.080041:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612703104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077394.080054:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077394.080095:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077394.080098:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077394.080105:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4382 00000100:00080000:2.0:1769077394.080111:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077394.080117:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077394.080126:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077394.080142:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612703232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077394.080257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077394.080357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff8b1005d5a300 x1855007612702848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.080374:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612702848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.080382:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.080386:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077394.080389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612702848:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077394.080408:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702976:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077394.080428:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077394.080438:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff8b1005d58e00 x1855007612702976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.080446:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612702976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.080452:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.080454:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077394.080456:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612702976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077394.080460:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077394.080472:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077394.080478:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff8b1005d5b480 x1855007612703104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.080486:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612703104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.080496:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.080498:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077394.080500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077394.080504:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077394.080515:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077394.080520:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff8b1005d5bb80 x1855007612703232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.080527:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612703232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.080533:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.080535:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077394.080537:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077394.975777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077394.975790:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077395.999786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077395.999847:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077397.023785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077397.023799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077398.047799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077398.047851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077399.071844:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077399.071900:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077399.199841:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077399.199851:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077399.199855:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077399.199865:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4387 00000100:00080000:3.0:1769077399.199876:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077399.199883:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077399.199917:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612703360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077399.199948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077399.199952:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077399.199954:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077399.199958:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4387 00000100:00080000:3.0:1769077399.199962:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077399.199965:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077399.199974:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077399.199986:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612703488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077399.199997:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077399.200000:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077399.200002:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077399.200006:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4387 00000100:00080000:3.0:1769077399.200010:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077399.200012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612703360:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077399.200017:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077399.200026:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077399.200037:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612703616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077399.200047:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077399.200051:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077399.200053:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077399.200056:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4387 00000100:00080000:3.0:1769077399.200060:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077399.200063:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077399.200068:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077399.200077:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612703744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077399.200090:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077399.200130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff8b10037ded80 x1855007612703360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077399.200150:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612703360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077399.200162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077399.200167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077399.200171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612703360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077399.200180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703488:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077399.200207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077399.200222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff8b10037de680 x1855007612703488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077399.200237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612703488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077399.200246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077399.200250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077399.200253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077399.200280:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077399.200305:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077399.200318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff8b10037dc380 x1855007612703616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077399.200332:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612703616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077399.200342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077399.200346:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077399.200350:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077399.200357:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077399.200379:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077399.200392:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff8b10037dd180 x1855007612703744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077399.200406:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612703744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077399.200415:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077399.200418:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077399.200422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612703744:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077400.095811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077400.095855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077401.119804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077401.119858:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077402.143832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077402.143897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077403.167759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077403.167810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077404.191731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077404.191796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077404.319749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077404.319760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077404.319764:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077404.319775:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4392 00000100:00080000:3.0:1769077404.319786:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077404.319793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077404.319825:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612703872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077404.319862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077404.319866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077404.319868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077404.319872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4392 00000100:00080000:3.0:1769077404.319877:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077404.319882:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077404.319891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077404.319906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612704000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077404.319915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077404.319918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077404.319920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077404.319925:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612703872:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077404.319925:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4392 00000100:00080000:3.0:1769077404.319929:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077404.319936:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077404.319940:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077404.319949:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612704128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077404.319956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077404.319958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077404.319960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077404.319962:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4392 00000100:00080000:3.0:1769077404.319964:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077404.319967:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077404.319970:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077404.319983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612704256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077404.319996:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077404.320029:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff8b10037dca80 x1855007612703872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077404.320049:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612703872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077404.320060:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077404.320065:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077404.320069:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612703872:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077404.320109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704000:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077404.320129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077404.320139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff8b10037dd180 x1855007612704000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077404.320148:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612704000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077404.320159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077404.320161:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077404.320163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077404.320168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077404.320179:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077404.320186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff8b10037dc380 x1855007612704128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077404.320196:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612704128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077404.320205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077404.320208:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077404.320211:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077404.320217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077404.320238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077404.320251:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff8b10037de680 x1855007612704256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077404.320263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612704256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077404.320269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077404.320271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077404.320274:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077405.215749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077405.215818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077406.239741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077406.239802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077407.263787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077407.263829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077408.287720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077408.287779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077409.311806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077409.311897:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077409.439806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077409.439819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077409.439825:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077409.439838:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4397 00000100:00080000:1.0:1769077409.439851:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077409.439860:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077409.439907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612704384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077409.439950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077409.439955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077409.439958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077409.439963:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4397 00000100:00080000:1.0:1769077409.439967:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077409.439972:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077409.439984:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077409.440002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612704512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077409.440018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077409.440027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769077409.440029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612704384:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077409.440031:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077409.440052:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4397 00000100:00080000:1.0:1769077409.440057:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077409.440063:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077409.440070:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077409.440091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612704640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077409.440102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077409.440106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077409.440109:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077409.440114:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4397 00000100:00080000:1.0:1769077409.440119:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077409.440124:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077409.440132:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077409.440143:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612704768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077409.440168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077409.440230:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff8b1006135880 x1855007612704384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.440250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612704384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.440261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.440265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077409.440269:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612704384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077409.440299:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704512:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077409.440320:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077409.440330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff8b1006137b80 x1855007612704512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.440340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612704512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.440350:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.440354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077409.440358:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704512:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077409.440365:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077409.440408:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077409.440422:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff8b1006135c00 x1855007612704640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.440438:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612704640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.440450:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.440453:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077409.440456:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077409.440462:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077409.440483:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077409.440493:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff8b1006136a00 x1855007612704768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.440522:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612704768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.440536:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.440539:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077409.440542:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612704768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077410.335742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077410.335830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077411.359780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077411.359867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077412.383726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077412.383766:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077413.407739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077413.407796:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077414.431764:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077414.431778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077414.559745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077414.559755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077414.559770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077414.559778:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4402 00000100:00080000:3.0:1769077414.559788:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077414.559793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077414.559831:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612704896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077414.559854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077414.559857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077414.559858:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077414.559861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4402 00000100:00080000:3.0:1769077414.559863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077414.559866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077414.559874:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077414.559883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612705024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077414.559891:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612704896:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077414.559905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077414.559908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077414.559910:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077414.559913:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4402 00000100:00080000:3.0:1769077414.559917:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077414.559920:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077414.559927:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077414.559939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612705152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077414.559949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077414.559953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769077414.559958:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077414.559958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077414.559966:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4402 00000100:00080000:3.0:1769077414.559971:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077414.559976:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077414.559982:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077414.559996:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff8b10037ded80 x1855007612704896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.559996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612705280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077414.560032:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612704896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077414.560041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077414.560045:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077414.560048:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612704896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077414.560067:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705024:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077414.560082:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077414.560090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff8b10037de300 x1855007612705024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077414.560099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612705024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077414.560104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077414.560107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077414.560109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705024:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077414.560113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077414.560123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077414.560138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff8b10037ddf80 x1855007612705152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077414.560146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612705152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077414.560151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077414.560153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077414.560155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077414.560158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077414.560168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077414.560174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff8b10037dd500 x1855007612705280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077414.560182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612705280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077414.560191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077414.560194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077414.560198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077415.455708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077415.455782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077416.479726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077416.479797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077417.503734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077417.503800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077418.527742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077418.527823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077419.551753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077419.551819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077419.679741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.679750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077419.679754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.679761:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4407 00000100:00080000:3.0:1769077419.679772:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.679814:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077419.679850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612705408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.679873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.679876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077419.679877:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.679880:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4407 00000100:00080000:3.0:1769077419.679882:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.679885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077419.679892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077419.679901:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612705536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.679908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.679911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077419.679912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.679915:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4407 00000100:00080000:3.0:1769077419.679917:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077419.679920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612705408:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077419.679922:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077419.679930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077419.679939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612705664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.679946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.679949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077419.679950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.679953:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4407 00000100:00080000:3.0:1769077419.679955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.679958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077419.679961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077419.679967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612705792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077419.679983:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077419.680014:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff8b10037dc000 x1855007612705408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077419.680028:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612705408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077419.680036:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077419.680040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077419.680043:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612705408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077419.680048:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705536:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077419.680061:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077419.680069:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff8b10037dd880 x1855007612705536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077419.680078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612705536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077419.680087:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077419.680090:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077419.680092:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077419.680110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077419.680136:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077419.680144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff8b10037df100 x1855007612705664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077419.680152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612705664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077419.680157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077419.680160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077419.680162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077419.680165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077419.680176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077419.680183:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff8b10037ddc00 x1855007612705792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077419.680190:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612705792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077419.680195:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077419.680197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077419.680200:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612705792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077420.575745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077420.575810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077421.599820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077421.599821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077422.623742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077422.623826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077423.647730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077423.647794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077424.671781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077424.671857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077424.799803:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077424.799811:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077424.799815:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077424.799822:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4412 00000100:00080000:2.0:1769077424.799832:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077424.799839:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077424.799879:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612705920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077424.799920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077424.799924:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077424.799925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077424.799929:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4412 00000100:00080000:2.0:1769077424.799933:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077424.799937:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077424.799946:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077424.799957:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612706048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077424.799973:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612705920:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077424.799977:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077424.799985:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077424.799986:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077424.799990:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4412 00000100:00080000:2.0:1769077424.799992:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077424.799994:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077424.799999:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077424.800009:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612706176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077424.800015:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077424.800018:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077424.800019:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077424.800022:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4412 00000100:00080000:2.0:1769077424.800024:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077424.800027:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077424.800030:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077424.800038:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612706304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077424.800065:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077424.800126:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff8b1005d59880 x1855007612705920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077424.800148:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612705920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077424.800162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077424.800179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077424.800184:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612705920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077424.800216:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706048:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077424.800237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077424.800263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff8b1005d59500 x1855007612706048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077424.800274:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612706048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077424.800291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077424.800295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077424.800299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706048:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077424.800307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077424.800333:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077424.800347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff8b1005d5ad80 x1855007612706176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077424.800359:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612706176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077424.800365:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077424.800368:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077424.800370:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077424.800375:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077424.800390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077424.800406:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff8b1005d58380 x1855007612706304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077424.800414:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612706304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077424.800420:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077424.800423:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077424.800425:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077425.695777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077425.695789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077426.719757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077426.719815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077427.743727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077427.743788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077428.767788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077428.767816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077429.791698:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077429.791787:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077429.919768:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077429.919778:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077429.919783:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077429.919795:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4418 00000100:00080000:2.0:1769077429.919807:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077429.919814:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077429.919850:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612706432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077429.919877:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077429.919882:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077429.919884:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077429.919888:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4418 00000100:00080000:2.0:1769077429.919893:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077429.919898:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077429.919908:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077429.919927:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612706560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077429.919931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612706432:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077429.919945:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077429.919949:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077429.919951:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077429.919955:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4418 00000100:00080000:2.0:1769077429.919959:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077429.919963:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077429.919968:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077429.919981:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612706688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077429.919988:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077429.919993:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077429.919994:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077429.919999:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4418 00000100:00080000:2.0:1769077429.920004:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077429.920008:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077429.920014:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077429.920028:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612706816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077429.920039:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077429.920074:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077429.920110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077430.815748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077430.815838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077431.839724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077431.839792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077432.863731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077432.863811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077432.991904:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077432.991977:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077432.991993:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077432.992031:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077432.992042:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077432.992048:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769077432.992094:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077432] req@ffff8b1005d5ad80 x1855007612706816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077432.992149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612706816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077432.992162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077432.992169:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077432.992173:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077432.992182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077432] req@ffff8b1005d5aa00 x1855007612706560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077432.992198:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612706560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077432.992207:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077432.992210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077432.992213:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077432.992220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077432] req@ffff8b1005d58380 x1855007612706688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077432.992232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612706688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077432.992248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077432.992252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077432.992256:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612706688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077432.992265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077432] req@ffff8b1005d5b100 x1855007612706432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077432.992280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612706432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077432.992288:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077432.992292:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077432.992296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612706432:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077433.887699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077433.887756:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077434.911742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077434.911892:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077435.039770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.039782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077435.039787:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.039797:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4423 00000100:00080000:3.0:1769077435.039808:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.039814:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077435.039846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612706944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.039894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.039898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.039901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.039905:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4423 00000100:00080000:3.0:1769077435.039909:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.039914:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077435.039924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.039939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612707072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.039948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.039952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077435.039954:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.039958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4423 00000100:00080000:3.0:1769077435.039962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.039966:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077435.039971:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.039981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612707200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077435.039991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077435.039994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077435.039996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077435.040000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4423 00000100:00080000:3.0:1769077435.040003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077435.040007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077435.040011:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077435.040020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612707328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077435.040110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612706944:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077435.040197:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077435.040237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff8b10037dea00 x1855007612706944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077435.040260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612706944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077435.040275:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077435.040280:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077435.040286:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612706944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077435.040295:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707072:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077435.040323:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077435.040340:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff8b10037dc700 x1855007612707072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077435.040356:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612707072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077435.040366:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077435.040369:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077435.040373:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707072:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077435.040381:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077435.040423:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077435.040437:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff8b10037df800 x1855007612707200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077435.040451:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612707200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077435.040460:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077435.040464:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077435.040468:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077435.040475:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077435.040512:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077435.040526:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077435/real 1769077435] req@ffff8b10037df480 x1855007612707328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077435.040539:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612707328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077435.040548:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077435.040552:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077435.040556:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077435.935751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077435.935781:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077436.959730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077436.959789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077437.983700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077437.983772:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077439.007743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077439.007795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077440.031753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077440.031863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077440.159790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.159802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077440.159807:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.159818:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4428 00000100:00080000:3.0:1769077440.159831:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.159838:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077440.159883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612707456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.159912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.159916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077440.159927:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.159932:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4428 00000100:00080000:3.0:1769077440.159936:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.159941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077440.159954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077440.159972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612707584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.159984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.160001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077440.160004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.160009:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4428 00000100:00080000:3.0:1769077440.160013:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077440.160019:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612707456:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077440.160022:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077440.160032:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077440.160047:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612707712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.160059:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.160062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077440.160063:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.160066:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4428 00000100:00080000:3.0:1769077440.160070:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.160073:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077440.160077:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077440.160086:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612707840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077440.160109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077440.160149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff8b10037dce00 x1855007612707456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077440.160165:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612707456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077440.160174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077440.160178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077440.160181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612707456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077440.160187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707584:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077440.160203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077440.160211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff8b10037df480 x1855007612707584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077440.160220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612707584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077440.160225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077440.160227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077440.160229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707584:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077440.160251:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077440.160267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077440.160279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff8b10037df800 x1855007612707712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077440.160292:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612707712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077440.160301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077440.160305:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077440.160309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077440.160315:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077440.160336:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077440.160349:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff8b10037dc700 x1855007612707840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077440.160361:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612707840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077440.160370:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077440.160374:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077440.160377:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612707840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077441.055742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077441.055827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077442.079721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077442.079834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077443.103808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077443.103842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077444.127746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077444.127850:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077445.151742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077445.151783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077445.279834:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077445.279846:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077445.279851:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077445.279861:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4433 00000100:00080000:2.0:1769077445.279873:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077445.279882:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077445.279958:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612707968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077445.279988:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077445.279993:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077445.279995:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077445.279999:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4433 00000100:00080000:2.0:1769077445.280004:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077445.280024:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077445.280035:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077445.280048:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612707968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077445.280053:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612708096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077445.280065:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077445.280070:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077445.280072:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077445.280076:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4433 00000100:00080000:2.0:1769077445.280081:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077445.280086:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077445.280092:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077445.280104:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612708224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077445.280119:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769077445.280123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769077445.280124:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077445.280127:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077445.280135:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4433 00000100:00080000:2.0:1769077445.280140:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077445.280145:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077445.280150:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077445.280167:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612708352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077445.280168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077445] req@ffff8b1005d59500 x1855007612707968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077445.280213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612707968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077445.280226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077445.280232:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077445.280237:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612707968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077445.280261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708096:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077445.280281:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077445.280299:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077445] req@ffff8b1005d59880 x1855007612708096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077445.280308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612708096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077445.280314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077445.280317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077445.280319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708096:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077445.280323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077445.280336:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077445.280343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077445] req@ffff8b1005d5bb80 x1855007612708224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077445.280350:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612708224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077445.280356:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077445.280357:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077445.280359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077445.280363:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077445.280374:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077445.280380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077445] req@ffff8b1005d5b480 x1855007612708352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077445.280387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612708352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077445.280393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077445.280395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077445.280396:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077446.175741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077446.175838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077447.199785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077447.199789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077448.223749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077448.223806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077449.247708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077449.247770:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077450.271772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077450.271794:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077450.399774:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077450.399783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077450.399787:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077450.399794:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4438 00000100:00080000:0.0:1769077450.399805:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077450.399810:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077450.399861:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612708480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077450.399884:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077450.399887:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077450.399889:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077450.399891:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4438 00000100:00080000:0.0:1769077450.399894:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077450.399896:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077450.399903:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077450.399913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612708608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077450.399925:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077450.399927:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077450.399928:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077450.399932:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4438 00000100:00080000:0.0:1769077450.399934:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077450.399936:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612708480:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077450.399938:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077450.399943:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077450.399951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612708736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077450.399957:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077450.399959:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077450.399961:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077450.399963:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4438 00000100:00080000:0.0:1769077450.399966:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077450.399968:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077450.399970:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077450.399978:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612708864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077450.400005:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077450.400035:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff8b1006026300 x1855007612708480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077450.400049:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612708480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077450.400057:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077450.400061:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077450.400065:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612708480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077450.400089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708608:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077450.400113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077450.400126:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff8b1006025880 x1855007612708608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077450.400139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612708608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077450.400145:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077450.400147:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077450.400149:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077450.400154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077450.400170:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077450.400177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff8b1006024000 x1855007612708736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077450.400185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612708736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077450.400190:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077450.400192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077450.400194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077450.400198:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077450.400208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077450.400222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff8b1006026a00 x1855007612708864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077450.400229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612708864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077450.400239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077450.400241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077450.400243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612708864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077451.295724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077451.295770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077452.319752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077452.319825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077453.343705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077453.343765:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077454.367774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077454.367844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077455.391770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077455.391847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077455.519828:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077455.519838:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077455.519843:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077455.519851:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4443 00000100:00080000:0.0:1769077455.519864:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077455.519870:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077455.519906:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612708992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077455.519926:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077455.519930:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077455.519931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077455.519935:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4443 00000100:00080000:0.0:1769077455.519938:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077455.519942:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077455.519950:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077455.519967:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612709120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077455.519979:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612708992:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077455.519983:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077455.519994:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077455.519996:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077455.519999:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4443 00000100:00080000:0.0:1769077455.520001:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077455.520004:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077455.520008:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077455.520021:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612709248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077455.520028:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077455.520031:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077455.520032:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077455.520035:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4443 00000100:00080000:0.0:1769077455.520037:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077455.520045:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077455.520049:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077455.520056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612709376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077455.520073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077455.520114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff8b1006026d80 x1855007612708992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077455.520130:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612708992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077455.520140:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077455.520144:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077455.520147:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612708992:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077455.520174:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709120:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077455.520190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077455.520200:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff8b1006027b80 x1855007612709120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077455.520211:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612709120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077455.520221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077455.520225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077455.520228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077455.520237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077455.520261:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077455.520274:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff8b1006025f80 x1855007612709248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077455.520284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612709248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077455.520290:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077455.520293:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077455.520295:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077455.520300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077455.520325:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077455.520333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff8b1006024a80 x1855007612709376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077455.520341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612709376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077455.520347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077455.520349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077455.520352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077456.415745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077456.415810:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077457.439809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077457.439837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077458.463750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077458.463812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077459.487806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077459.487876:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077460.511750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077460.511826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077460.639845:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077460.639868:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077460.639874:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077460.639885:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4448 00000100:00080000:2.0:1769077460.639900:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077460.639907:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077460.639954:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612709504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077460.639993:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077460.639997:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077460.639998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077460.640002:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4448 00000100:00080000:2.0:1769077460.640004:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077460.640007:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077460.640015:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077460.640025:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612709632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077460.640033:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077460.640036:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077460.640037:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077460.640039:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4448 00000100:00080000:2.0:1769077460.640042:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077460.640044:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077460.640049:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077460.640056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612709760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077460.640068:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077460.640070:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077460.640072:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077460.640074:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4448 00000100:00080000:2.0:1769077460.640076:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077460.640079:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077460.640082:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077460.640093:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612709888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077460.640095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612709504:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077460.640186:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077460.640226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff8b1005d58e00 x1855007612709504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077460.640263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612709504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077460.640277:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077460.640282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077460.640287:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612709504:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077460.640297:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709632:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077460.640341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077460.640353:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff8b1005d5a300 x1855007612709632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077460.640367:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612709632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077460.640376:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077460.640380:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077460.640384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709632:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077460.640390:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077460.640409:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077460.640421:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff8b1005d58000 x1855007612709760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077460.640433:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612709760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077460.640448:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077460.640451:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077460.640455:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077460.640498:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077460.640521:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077460.640534:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff8b1005d58a80 x1855007612709888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077460.640546:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612709888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077460.640557:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077460.640560:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077460.640564:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612709888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077461.535748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077461.535828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077462.559735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077462.559795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077463.583748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077463.583805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077464.607779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077464.607799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077465.631742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077465.631813:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077465.759716:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077465.759727:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077465.759731:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077465.759740:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4453 00000100:00080000:2.0:1769077465.759749:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077465.759755:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077465.759787:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612710016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077465.759811:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077465.759815:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077465.759817:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077465.759821:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4453 00000100:00080000:2.0:1769077465.759825:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077465.759829:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077465.759837:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077465.759847:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612710144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077465.759862:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077465.759865:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077465.759867:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077465.759870:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4453 00000100:00080000:2.0:1769077465.759873:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077465.759876:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077465.759881:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077465.759893:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612710272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077465.759903:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077465.759907:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077465.759910:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077465.759915:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4453 00000100:00080000:2.0:1769077465.759920:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077465.759925:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077465.759932:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077465.759939:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612710016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077465.759950:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612710400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077465.760054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077465.760082:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff8b1005d5b800 x1855007612710016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077465.760096:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612710016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077465.760104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077465.760107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077465.760110:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612710016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077465.760212:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710144:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077465.760234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077465.760247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff8b1005d59180 x1855007612710144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077465.760259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612710144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077465.760331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077465.760335:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077465.760338:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710144:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077465.760343:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077465.760361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077465.760370:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff8b1005d58a80 x1855007612710272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077465.760379:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612710272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077465.760385:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077465.760387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077465.760389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077465.760409:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077465.760421:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077465.760427:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff8b1005d58000 x1855007612710400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077465.760436:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612710400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077465.760444:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077465.760447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077465.760451:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077466.655733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077466.655807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077467.679746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077467.679830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077468.703721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077468.703821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077469.727743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077469.727821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077470.751804:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077470.751866:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077470.879778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077470.879788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077470.879792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077470.879800:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4458 00000100:00080000:3.0:1769077470.879810:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077470.879815:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077470.879847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612710528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077470.879877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077470.879881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077470.879884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077470.879888:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4458 00000100:00080000:3.0:1769077470.879892:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077470.879897:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077470.879907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077470.879921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612710656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077470.879931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077470.879934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077470.879936:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077470.879939:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4458 00000100:00080000:3.0:1769077470.879942:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077470.879944:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612710528:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077470.879945:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077470.879952:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077470.879960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612710784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077470.879967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077470.879970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077470.879971:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077470.879974:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4458 00000100:00080000:3.0:1769077470.879976:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077470.879978:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077470.879981:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077470.879990:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612710912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077470.880021:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077470.880058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff8b10037dea00 x1855007612710528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077470.880078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612710528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077470.880089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077470.880094:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077470.880099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612710528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077470.880124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710656:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077470.880147:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077470.880161:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff8b10037ddc00 x1855007612710656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077470.880176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612710656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077470.880185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077470.880189:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077470.880193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710656:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077470.880200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077470.880222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077470.880245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff8b10037df100 x1855007612710784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077470.880259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612710784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077470.880268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077470.880272:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077470.880276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077470.880282:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077470.880313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077470.880327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff8b10037dd880 x1855007612710912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077470.880342:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612710912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077470.880352:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077470.880355:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077470.880359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612710912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077471.775806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077471.775862:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077472.799733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077472.799828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077473.823734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077473.823788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077474.847697:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077474.847761:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077475.871747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077475.871797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077475.999777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077475.999788:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077475.999793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077475.999803:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4464 00000100:00080000:0.0:1769077475.999813:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077475.999821:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077475.999856:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612711040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077475.999886:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077475.999890:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077475.999892:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077475.999897:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4464 00000100:00080000:0.0:1769077475.999901:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077475.999906:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077475.999916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077475.999927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612711040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077475.999932:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612711168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077475.999946:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077475.999950:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077475.999953:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077475.999957:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4464 00000100:00080000:0.0:1769077475.999962:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077475.999967:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077475.999972:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077475.999984:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612711296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077475.999988:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077476.000000:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077476.000004:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077476.000007:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077476.000012:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4464 00000100:00080000:0.0:1769077476.000017:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077476.000021:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077476.000024:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077476] req@ffff8b1006024e00 x1855007612711040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077476.000029:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077476.000043:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612711040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077476.000047:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612711424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077476.000059:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077476.000067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077476.000071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612711040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077476.000091:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711168:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077476.000105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077476.000113:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077476] req@ffff8b1006024380 x1855007612711168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077476.000125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612711168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077476.000140:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077476.000144:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077476.000148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077476.000154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077476.000176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077476.000189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077476] req@ffff8b1006027480 x1855007612711296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077476.000201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612711296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077476.000207:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077476.000209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077476.000211:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077476.000215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077476.000230:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077476.000238:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077476] req@ffff8b1006024700 x1855007612711424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077476.000251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612711424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077476.000259:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077476.000263:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077476.000266:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077476.895751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077476.895810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077477.919748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077477.919833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077478.943824:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077478.943867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077479.967743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077479.967825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077480.991765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077480.991831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077481.119790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.119798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077481.119802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.119810:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4469 00000100:00080000:3.0:1769077481.119820:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.119826:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077481.119857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612711552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.119879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.119882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.119883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.119886:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4469 00000100:00080000:3.0:1769077481.119888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.119891:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077481.119898:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.119907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612711680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.119914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.119916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.119918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.119925:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4469 00000100:00100000:1.0:1769077481.119926:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612711552:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077481.119927:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.119933:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077481.119938:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.119946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612711808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.119953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.119956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077481.119957:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.119960:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4469 00000100:00080000:3.0:1769077481.119962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.119965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077481.119968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.119974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612711936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077481.119988:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077481.120018:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff8b10037dc000 x1855007612711552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077481.120032:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612711552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077481.120041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077481.120045:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077481.120049:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612711552:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077481.120058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711680:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077481.120083:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077481.120097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff8b10037dd500 x1855007612711680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077481.120110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612711680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077481.120128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077481.120135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077481.120139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077481.120161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077481.120178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077481.120187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff8b10037ddf80 x1855007612711808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077481.120195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612711808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077481.120201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077481.120203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077481.120205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077481.120209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077481.120219:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077481.120226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff8b10037de300 x1855007612711936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077481.120234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612711936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077481.120239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077481.120241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077481.120242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612711936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077482.015727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077482.015809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077483.039731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077483.039792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077484.063728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077484.063778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077485.087750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077485.087805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077486.111774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077486.111812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077486.239794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.239814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077486.239818:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.239828:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4474 00000100:00080000:1.0:1769077486.239839:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.239846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077486.239882:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612712064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.239918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.239922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077486.239925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.239929:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4474 00000100:00080000:1.0:1769077486.239934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.239939:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077486.239949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077486.239953:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612712064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077486.239964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612712192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.239976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.239980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077486.239982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.239985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4474 00000100:00080000:1.0:1769077486.239990:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.239994:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077486.239999:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077486.240011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612712320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.240021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.240025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077486.240027:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077486.240031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077486.240036:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4474 00000100:00080000:1.0:1769077486.240041:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.240046:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077486.240051:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077486.240064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612712448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077486.240074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff8b1006135180 x1855007612712064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077486.240091:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612712064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077486.240100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077486.240103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077486.240107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612712064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077486.240127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712192:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077486.240144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077486.240153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff8b1006137800 x1855007612712192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077486.240161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612712192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077486.240167:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077486.240169:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077486.240171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712192:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077486.240185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077486.240201:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077486.240208:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff8b1006136300 x1855007612712320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077486.240239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612712320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077486.240245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077486.240247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077486.240249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077486.240253:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077486.240268:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077486.240276:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff8b1006136a00 x1855007612712448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077486.240284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612712448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077486.240289:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077486.240291:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077486.240293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077487.135753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077487.135809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077488.159727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077488.159826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077489.183770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077489.183828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077490.207713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077490.207781:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077491.231756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077491.231815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077491.359767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077491.359776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077491.359780:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077491.359789:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4479 00000100:00080000:3.0:1769077491.359799:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077491.359804:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077491.359836:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612712576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077491.359861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077491.359864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077491.359865:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077491.359868:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4479 00000100:00080000:3.0:1769077491.359871:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077491.359873:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077491.359880:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077491.359889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612712704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077491.359901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077491.359906:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612712576:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077491.359907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077491.359910:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077491.359920:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4479 00000100:00080000:3.0:1769077491.359924:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077491.359928:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077491.359934:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077491.359948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612712832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077491.359966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077491.359970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077491.359972:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077491.359975:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4479 00000100:00080000:3.0:1769077491.359978:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077491.359984:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077491.359985:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077491.359993:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077491.360002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612712960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077491.360026:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff8b10037ded80 x1855007612712576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077491.360042:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612712576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077491.360050:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077491.360053:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077491.360056:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612712576:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077491.360075:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712704:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077491.360092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077491.360101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff8b10037de680 x1855007612712704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077491.360110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612712704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077491.360116:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077491.360120:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077491.360123:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712704:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077491.360142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077491.360165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077491.360177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff8b10037dc380 x1855007612712832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077491.360191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612712832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077491.360200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077491.360204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077491.360207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077491.360213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077491.360229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077491.360237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff8b10037dd180 x1855007612712960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077491.360245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612712960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077491.360251:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077491.360253:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077491.360255:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612712960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077492.255705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077492.255784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077493.279806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077493.279806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077494.303708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077494.303785:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077495.327720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077495.327809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077496.351748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077496.351806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077496.479769:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077496.479778:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077496.479782:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077496.479789:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4484 00000100:00080000:0.0:1769077496.479799:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077496.479804:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077496.479841:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612713088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077496.479873:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077496.479878:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077496.479880:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077496.479884:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4484 00000100:00080000:0.0:1769077496.479888:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077496.479891:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077496.479899:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077496.479910:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612713216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077496.479919:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077496.479930:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077496.479933:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077496.479933:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612713088:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077496.479938:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4484 00000100:00080000:0.0:1769077496.479943:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077496.479945:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077496.479949:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077496.479959:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612713344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077496.479968:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077496.479970:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077496.479972:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077496.479975:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4484 00000100:00080000:0.0:1769077496.479986:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077496.479989:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077496.479992:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077496.480000:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612713472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077496.480026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077496.480078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077496.480099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077497.375725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077497.375795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077498.399755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077498.399809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077499.423745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077499.423812:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077499.551895:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077499.551942:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077499.551958:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077499.552030:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077499.552043:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077499.552059:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769077499.552073:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077499] req@ffff8b1006024380 x1855007612713344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077499.552099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612713344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077499.552111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077499.552116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077499.552121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077499.552130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077499] req@ffff8b1006024e00 x1855007612713472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077499.552142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612713472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077499.552150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077499.552153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077499.552157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077499.552164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077499] req@ffff8b1006024700 x1855007612713088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077499.552177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612713088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077499.552187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077499.552191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077499.552194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612713088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077499.552201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077499] req@ffff8b1006027480 x1855007612713216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077499.552214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612713216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077499.552230:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077499.552234:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077499.552238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077500.447799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077500.447821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077501.471741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077501.471835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077501.599778:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077501.599790:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077501.599794:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077501.599805:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4489 00000100:00080000:2.0:1769077501.599816:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077501.599825:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077501.599862:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612713600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077501.599885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077501.599889:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077501.599892:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077501.599897:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4489 00000100:00080000:2.0:1769077501.599902:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077501.599906:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077501.599914:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612713600:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769077501.599918:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077501.599936:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612713728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077501.599949:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077501.599967:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077501.599971:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077501.599978:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4489 00000100:00080000:2.0:1769077501.599982:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077501.599988:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077501.599996:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077501.600005:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077501.600014:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612713856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077501.600027:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077501.600032:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077501.600035:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077501.600040:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4489 00000100:00080000:2.0:1769077501.600045:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077501.600050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff8b1005d5a300 x1855007612713600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077501.600054:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077501.600064:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077501.600080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612713600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077501.600083:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612713984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077501.600121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077501.600129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077501.600134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612713600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077501.600164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713728:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077501.600192:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077501.600205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff8b1005d58e00 x1855007612713728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077501.600219:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612713728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077501.600227:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077501.600231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077501.600250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713728:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077501.600258:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077501.600278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077501.600290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff8b1005d5b480 x1855007612713856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077501.600303:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612713856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077501.600311:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077501.600314:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077501.600317:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077501.600323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077501.600342:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077501.600353:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff8b1005d5bb80 x1855007612713984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077501.600372:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612713984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077501.600380:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077501.600384:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077501.600387:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612713984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077502.495782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077502.495834:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077503.519765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077503.519804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077504.543763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077504.543836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077505.567748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077505.567800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077506.591761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077506.591862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077506.719780:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077506.719792:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077506.719796:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077506.719806:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4494 00000100:00080000:0.0:1769077506.719818:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077506.719825:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077506.719863:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612714112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077506.719927:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077506.719932:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077506.719934:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077506.719939:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4494 00000100:00080000:0.0:1769077506.719944:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077506.719949:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077506.719959:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077506.719968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612714112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077506.719977:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612714240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077506.719991:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077506.719995:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077506.719998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077506.720003:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4494 00000800:00000100:1.0:1769077506.720024:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077506.720025:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077506.720033:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077506.720040:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077506.720056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612714368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077506.720057:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff8b1006024a80 x1855007612714112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.720091:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077506.720094:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612714112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.720099:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077506.720105:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077506.720110:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.720114:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4494 00000100:00080000:0.0:1769077506.720120:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077506.720125:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:1.0:1769077506.720132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000008:00100000:0.0:1769077506.720133:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077506.720141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612714112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077506.720152:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612714496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077506.720172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714240:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077506.720195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077506.720205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff8b1006025f80 x1855007612714240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077506.720214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612714240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077506.720225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077506.720228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077506.720230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714240:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077506.720234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077506.720246:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077506.720252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff8b1006027b80 x1855007612714368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077506.720260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612714368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077506.720266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077506.720268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077506.720270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077506.720273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077506.720284:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077506.720290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff8b1006026d80 x1855007612714496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077506.720297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612714496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077506.720302:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077506.720304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077506.720307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077507.615752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077507.615797:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077508.639756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077508.639809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077509.663762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077509.663817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077510.687826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077510.687868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077511.711740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077511.711781:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077511.839798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.839808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077511.839812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.839819:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4499 00000100:00080000:3.0:1769077511.839829:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.839834:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077511.839868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612714624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.839894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.839896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077511.839898:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.839900:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4499 00000100:00080000:3.0:1769077511.839903:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.839905:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077511.839912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077511.839922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612714752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.839930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.839933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077511.839934:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.839936:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4499 00000100:00080000:3.0:1769077511.839938:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.839941:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077511.839945:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077511.839954:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612714624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077511.839954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612714880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.839970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.839972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077511.839973:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.839975:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4499 00000100:00080000:3.0:1769077511.839977:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.839979:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077511.839983:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077511.839990:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612715008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077511.840034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077511.840064:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077511] req@ffff8b10037dca80 x1855007612714624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077511.840078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612714624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077511.840086:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077511.840089:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077511.840092:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612714624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077511.840098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714752:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077511.840113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077511.840124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077511] req@ffff8b10037dd180 x1855007612714752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077511.840138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612714752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077511.840147:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077511.840150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077511.840154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077511.840177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077511.840199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077511.840209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077511] req@ffff8b10037dc380 x1855007612714880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077511.840217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612714880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077511.840223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077511.840225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077511.840227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612714880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077511.840231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077511.840243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077511.840249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077511] req@ffff8b10037de680 x1855007612715008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077511.840256:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612715008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077511.840261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077511.840263:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077511.840265:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077512.735754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077512.735826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077513.759710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077513.759784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077514.783829:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077514.783859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077515.807702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077515.807835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077516.831738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077516.831796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077516.959797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077516.959806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077516.959810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077516.959818:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4505 00000100:00080000:1.0:1769077516.959828:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077516.959833:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077516.959867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612715136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077516.959897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077516.959901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077516.959902:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077516.959905:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4505 00000100:00080000:1.0:1769077516.959908:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077516.959913:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077516.959923:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077516.959938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612715264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077516.959952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077516.959958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077516.959964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077516.959964:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612715136:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077516.959986:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4505 00000100:00080000:1.0:1769077516.959990:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077516.959996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077516.960003:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077516.960017:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612715392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077516.960024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077516.960027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077516.960029:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077516.960032:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4505 00000100:00080000:1.0:1769077516.960035:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077516.960038:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077516.960043:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077516.960049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612715520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077516.960067:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077516.960108:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff8b1006135c00 x1855007612715136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077516.960128:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612715136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077516.960141:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077516.960147:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077516.960157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612715136:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077516.960196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715264:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077516.960221:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077516.960232:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff8b1006137b80 x1855007612715264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077516.960242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612715264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077516.960248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077516.960251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077516.960253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715264:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077516.960259:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077516.960272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077516.960278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff8b1006135880 x1855007612715392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077516.960287:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612715392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077516.960292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077516.960294:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077516.960296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077516.960300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077516.960312:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077516.960318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff8b1006135500 x1855007612715520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077516.960327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612715520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077516.960332:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077516.960334:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077516.960335:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077517.855774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077517.855849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077518.879734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077518.879794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077519.903755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077519.903814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077520.927776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077520.927857:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077521.951813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077521.951851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077522.079750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077522.079759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077522.079762:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077522.079770:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4510 00000100:00080000:3.0:1769077522.079779:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077522.079785:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077522.079818:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612715648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077522.079843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077522.079846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077522.079847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077522.079850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4510 00000100:00080000:3.0:1769077522.079853:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077522.079856:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077522.079864:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077522.079873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612715776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077522.079879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612715648:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077522.079883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077522.079888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077522.079889:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077522.079892:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4510 00000100:00080000:3.0:1769077522.079894:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077522.079897:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077522.079900:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077522.079909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612715904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077522.079915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077522.079917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077522.079918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077522.079930:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4510 00000100:00080000:3.0:1769077522.079932:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077522.079935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077522.079938:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077522.079940:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077522.079950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612716032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077522.079975:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff8b10037ded80 x1855007612715648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.079998:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612715648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077522.080006:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077522.080010:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077522.080013:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612715648:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077522.080031:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715776:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077522.080047:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077522.080056:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff8b10037de300 x1855007612715776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.080064:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612715776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077522.080070:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077522.080073:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077522.080076:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077522.080080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077522.080092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077522.080098:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff8b10037ddf80 x1855007612715904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.080106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612715904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077522.080112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077522.080114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077522.080116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612715904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077522.080119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077522.080130:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077522.080141:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff8b10037dd500 x1855007612716032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077522.080154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612716032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077522.080163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077522.080166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077522.080170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077522.975774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077522.975818:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077523.999742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077523.999820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077525.023725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077525.023761:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077526.047756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077526.047820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077527.071731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077527.071809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077527.199777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077527.199787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077527.199790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077527.199798:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4515 00000100:00080000:1.0:1769077527.199807:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077527.199813:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077527.199846:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612716160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077527.199871:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077527.199874:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077527.199876:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077527.199879:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4515 00000100:00080000:1.0:1769077527.199881:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077527.199883:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077527.199891:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077527.199899:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612716288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077527.199906:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077527.199909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077527.199910:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077527.199913:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4515 00000100:00080000:1.0:1769077527.199915:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077527.199918:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077527.199921:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077527.199930:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612716416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077527.199932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612716160:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077527.199946:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077527.199949:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077527.199950:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077527.199953:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4515 00000100:00080000:1.0:1769077527.199956:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077527.199958:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077527.199961:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077527.199968:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612716544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077527.200006:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077527.200041:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff8b1006136d80 x1855007612716160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077527.200061:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612716160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.200073:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077527.200078:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077527.200082:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612716160:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077527.200090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716288:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077527.200112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077527.200125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff8b1006136680 x1855007612716288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077527.200139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612716288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.200148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077527.200151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077527.200155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716288:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077527.200177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077527.200208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077527.200221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff8b1006137100 x1855007612716416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077527.200235:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612716416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.200244:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077527.200247:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077527.200250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077527.200256:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077527.200276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077527.200286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff8b1006134e00 x1855007612716544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077527.200300:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612716544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.200308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077527.200312:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077527.200315:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077528.095741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077528.095818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077529.119785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077529.119799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077530.143762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077530.143827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077531.167721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077531.167763:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077532.191748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077532.191820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077532.319766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077532.319774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077532.319778:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077532.319786:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4520 00000100:00080000:3.0:1769077532.319796:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077532.319801:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077532.319834:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612716672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077532.319859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077532.319863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077532.319865:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077532.319869:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4520 00000100:00080000:3.0:1769077532.319873:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077532.319878:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077532.319888:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077532.319893:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612716672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077532.319900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612716800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077532.319909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077532.319913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077532.319914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077532.319918:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4520 00000100:00080000:3.0:1769077532.319922:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077532.319925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077532.319929:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077532.319938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612716928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077532.319946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077532.319950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077532.319952:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077532.319953:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077532.319955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4520 00000100:00080000:3.0:1769077532.319958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077532.319961:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077532.319965:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077532.319974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612717056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077532.319986:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff8b10037dc000 x1855007612716672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077532.320018:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612716672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077532.320030:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077532.320034:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077532.320038:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612716672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077532.320064:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716800:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077532.320089:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077532.320101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff8b10037dd880 x1855007612716800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077532.320112:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612716800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077532.320121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077532.320124:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077532.320128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716800:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077532.320135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077532.320157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077532.320167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff8b10037df100 x1855007612716928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077532.320176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612716928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077532.320181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077532.320183:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077532.320186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612716928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077532.320189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077532.320202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077532.320208:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff8b10037ddc00 x1855007612717056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077532.320216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612717056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077532.320225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077532.320228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077532.320230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077533.215745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077533.215804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077534.239748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077534.239829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077535.263755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077535.263814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077536.287810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077536.287823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077537.311737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077537.311809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077537.439749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.439758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077537.439762:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.439771:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4525 00000100:00080000:3.0:1769077537.439781:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.439788:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077537.439819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612717184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077537.439844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.439847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077537.439849:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.439853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4525 00000100:00080000:3.0:1769077537.439857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.439861:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077537.439869:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077537.439875:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612717184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077537.439883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612717312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077537.439893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.439896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077537.439897:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.439901:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4525 00000100:00080000:3.0:1769077537.439904:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.439908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077537.439912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077537.439923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612717440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077537.439931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.439933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077537.439935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.439938:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4525 00000100:00080000:3.0:1769077537.439942:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077537.439946:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077537.439948:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077537.439954:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077537.439962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612717568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077537.439985:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff8b10037dea00 x1855007612717184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077537.440001:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612717184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077537.440010:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077537.440013:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077537.440017:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612717184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077537.440035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717312:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077537.440053:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077537.440062:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff8b10037dc700 x1855007612717312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077537.440072:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612717312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077537.440079:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077537.440082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077537.440084:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717312:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077537.440088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077537.440100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077537.440106:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff8b10037df800 x1855007612717440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077537.440114:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612717440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077537.440120:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077537.440122:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077537.440125:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077537.440129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077537.440145:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077537.440157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff8b10037df480 x1855007612717568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077537.440170:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612717568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077537.440180:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077537.440184:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077537.440189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077538.335770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077538.335832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077539.359772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077539.359833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077540.383791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077540.383859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077541.407758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077541.407826:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077542.431729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077542.431810:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077542.559737:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077542.559746:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077542.559751:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077542.559762:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4530 00000100:00080000:2.0:1769077542.559773:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077542.559781:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077542.559826:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612717696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077542.559851:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077542.559854:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077542.559857:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077542.559864:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4530 00000100:00080000:2.0:1769077542.559867:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077542.559870:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077542.559877:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077542.559887:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612717824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077542.559900:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077542.559903:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077542.559904:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077542.559907:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4530 00000100:00080000:2.0:1769077542.559910:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077542.559913:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077542.559917:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077542.559922:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612717696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077542.559929:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612717952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077542.559938:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077542.559941:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077542.559942:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077542.559945:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4530 00000100:00080000:2.0:1769077542.559948:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077542.559951:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077542.559954:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077542.559962:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612718080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077542.559979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077542.560010:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff8b1005d59880 x1855007612717696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077542.560022:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612717696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077542.560030:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077542.560033:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077542.560036:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612717696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077542.560041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717824:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077542.560055:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077542.560063:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff8b1005d59500 x1855007612717824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077542.560071:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612717824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077542.560077:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077542.560079:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077542.560082:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717824:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077542.560101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077542.560112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077542.560122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff8b1005d5b100 x1855007612717952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077542.560135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612717952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077542.560144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077542.560147:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077542.560151:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612717952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077542.560157:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077542.560177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077542.560189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff8b1005d58380 x1855007612718080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077542.560202:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612718080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077542.560211:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077542.560215:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077542.560218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718080:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769077543.455776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077543.455800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077544.479721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077544.479776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077545.503753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077545.503804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077546.527775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077546.527877:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077547.551762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077547.551859:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077547.679786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077547.679795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077547.679799:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077547.679807:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4535 00000100:00080000:3.0:1769077547.679817:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077547.679822:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077547.679854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612718208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077547.679877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077547.679880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077547.679882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077547.679885:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4535 00000100:00080000:3.0:1769077547.679887:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077547.679889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077547.679896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077547.679905:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612718336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077547.679911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077547.679914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077547.679915:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077547.679918:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4535 00000100:00080000:3.0:1769077547.679919:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077547.679921:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077547.679924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077547.679930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612718464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077547.679942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077547.679944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077547.679945:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077547.679948:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4535 00000100:00080000:3.0:1769077547.679949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077547.679952:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077547.679955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077547.679961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612718592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077547.679968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612718208:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077547.680031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077547.680060:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff8b10037dce00 x1855007612718208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077547.680076:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612718208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077547.680083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077547.680087:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077547.680090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612718208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077547.680095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718336:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077547.680110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077547.680117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff8b10037df480 x1855007612718336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077547.680126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612718336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077547.680131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077547.680133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077547.680135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718336:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077547.680138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077547.680150:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077547.680156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff8b10037df800 x1855007612718464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077547.680163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612718464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077547.680173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077547.680175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077547.680177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077547.680222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077547.680237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077547.680245:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff8b10037dc700 x1855007612718592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077547.680252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612718592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077547.680258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077547.680260:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077547.680262:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077548.575708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077548.575774:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077549.599767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077549.599848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077550.623755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077550.623799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077551.647741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077551.647799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077552.671734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077552.671819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077552.799775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077552.799784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077552.799788:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077552.799796:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4540 00000100:00080000:1.0:1769077552.799806:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077552.799812:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077552.799846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612718720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077552.799875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077552.799879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077552.799880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077552.799883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4540 00000100:00080000:1.0:1769077552.799886:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077552.799888:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077552.799896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077552.799908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612718848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077552.799916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077552.799919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077552.799920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077552.799922:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4540 00000100:00080000:1.0:1769077552.799926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077552.799931:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077552.799936:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077552.799947:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612718720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077552.799952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612718976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077552.799969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077552.799974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077552.799977:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077552.799983:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4540 00000100:00080000:1.0:1769077552.799988:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077552.799994:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077552.800000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077552.800011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612719104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077552.800034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077552.800074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff8b1006134700 x1855007612718720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077552.800090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612718720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077552.800119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077552.800123:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077552.800126:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612718720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077552.800133:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718848:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077552.800154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077552.800164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff8b1006137480 x1855007612718848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077552.800178:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612718848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077552.800185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077552.800187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077552.800189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077552.800214:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077552.800252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077552.800266:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff8b1006134a80 x1855007612718976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077552.800281:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612718976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077552.800291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077552.800296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077552.800300:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612718976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077552.800308:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077552.800331:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077552.800341:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff8b1006134e00 x1855007612719104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077552.800350:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612719104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077552.800362:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077552.800364:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077552.800367:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077553.695757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077553.695821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077554.719755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077554.719823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077555.743757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077555.743839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077556.767753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077556.767823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077557.791815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077557.791817:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077557.919800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.919812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077557.919817:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.919828:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4546 00000100:00080000:3.0:1769077557.919839:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.919846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077557.919885:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612719232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.919914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.919917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077557.919918:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.919922:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4546 00000100:00080000:3.0:1769077557.919925:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.919929:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077557.919935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077557.919946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612719360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.919954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.919957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077557.919958:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.919960:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4546 00000100:00080000:3.0:1769077557.919962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077557.919968:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612719232:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077557.919968:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077557.919972:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077557.919980:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612719488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.919987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.919989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077557.919991:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.919993:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4546 00000100:00080000:3.0:1769077557.919995:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.919998:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077557.920001:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077557.920036:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612719616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077557.920045:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077557.920082:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff8b10037dea00 x1855007612719232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077557.920098:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612719232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077557.920106:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077557.920110:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077557.920113:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612719232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077557.920118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719360:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077557.920134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077557.920143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff8b10037ddc00 x1855007612719360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077557.920151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612719360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077557.920157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077557.920159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077557.920162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719360:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077557.920185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077557.920207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077557.920219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff8b10037df100 x1855007612719488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077557.920233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612719488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077557.920239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077557.920241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077557.920243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077557.920248:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077557.920267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077557.920279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff8b10037dd880 x1855007612719616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077557.920293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612719616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077557.920311:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077557.920315:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077557.920318:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077558.815710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077558.815753:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077559.839740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077559.839810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077560.863766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077560.863833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077561.887753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077561.887807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077562.911748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077562.911802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077563.039775:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077563.039787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077563.039793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077563.039807:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4551 00000100:00080000:2.0:1769077563.039821:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077563.039828:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077563.039874:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612719744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077563.039907:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077563.039911:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077563.039912:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077563.039915:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4551 00000100:00080000:2.0:1769077563.039918:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077563.039942:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077563.039952:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077563.039963:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612719872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077563.039974:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077563.039978:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077563.039981:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077563.039987:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4551 00000100:00080000:2.0:1769077563.039993:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077563.039996:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612719744:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077563.039998:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077563.040008:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077563.040020:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612720000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077563.040028:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077563.040031:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077563.040032:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077563.040035:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4551 00000100:00080000:2.0:1769077563.040037:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077563.040040:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077563.040043:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077563.040053:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612720128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077563.040127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077563.040164:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077563.040191:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077563.935791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077563.935894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077564.959825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077564.959860:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077565.983716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077565.983788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077566.112241:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077566.112310:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077566.112345:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077566.112394:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077566.112409:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077566.112428:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769077566.112447:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077566] req@ffff8b1005d5ad80 x1855007612719872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077566.112482:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612719872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077566.112495:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077566.112501:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077566.112505:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612719872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077566.112514:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077566] req@ffff8b1005d58380 x1855007612720000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077566.112528:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612720000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077566.112536:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077566.112540:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077566.112545:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077566.112553:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077566] req@ffff8b1005d5b100 x1855007612720128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077566.112565:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612720128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077566.112574:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077566.112577:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077566.112581:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077566.112590:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077563/real 1769077566] req@ffff8b1005d5aa00 x1855007612719744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077566.112603:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612719744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077566.112610:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077566.112614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077566.112617:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612719744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077567.007741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077567.007891:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077568.031726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077568.031790:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077568.159780:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077568.159789:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077568.159793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077568.159801:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4556 00000100:00080000:1.0:1769077568.159811:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077568.159818:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077568.159892:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612720256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077568.159934:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077568.159939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077568.159942:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077568.159948:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4556 00000100:00080000:1.0:1769077568.159953:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077568.159975:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077568.159984:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077568.160000:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612720384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077568.160004:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612720256:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077568.160018:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077568.160023:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077568.160025:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077568.160030:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4556 00000100:00080000:1.0:1769077568.160035:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077568.160040:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077568.160045:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077568.160056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612720512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077568.160063:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077568.160065:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077568.160067:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077568.160069:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4556 00000100:00080000:1.0:1769077568.160073:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077568.160077:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769077568.160081:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769077568.160084:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077568.160099:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612720640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077568.160118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff8b1006137100 x1855007612720256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077568.160139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612720256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077568.160147:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077568.160150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077568.160153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612720256:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077568.160175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720384:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077568.160202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077568.160215:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff8b1006136680 x1855007612720384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077568.160229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612720384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077568.160238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077568.160241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077568.160244:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077568.160250:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077568.160270:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077568.160282:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff8b1006136d80 x1855007612720512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077568.160295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612720512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077568.160309:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077568.160313:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077568.160316:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077568.160322:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077568.160361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077568.160372:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff8b1006135500 x1855007612720640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077568.160385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612720640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077568.160392:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077568.160395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077568.160398:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077569.055750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077569.055834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077570.079714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077570.079799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077571.103742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077571.103850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077572.127746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077572.127839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077573.151809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077573.151919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077573.279811:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.279826:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077573.279832:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.279847:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4561 00000100:00080000:3.0:1769077573.279863:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.279872:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077573.279922:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612720768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.279955:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.279960:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077573.279962:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.279966:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4561 00000100:00080000:3.0:1769077573.279970:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.279974:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077573.279985:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077573.279998:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612720896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.280014:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.280017:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077573.280021:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077573.280026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612720768:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077573.280026:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4561 00000100:00080000:3.0:1769077573.280032:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.280042:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077573.280047:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077573.280061:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612721024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.280069:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.280072:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077573.280074:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.280077:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4561 00000100:00080000:3.0:1769077573.280079:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.280083:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769077573.280121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769077573.280161:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077573.280178:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612721152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077573.280181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff8b10037dc000 x1855007612720768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077573.280221:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612720768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077573.280234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077573.280238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077573.280243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612720768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077573.280270:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720896:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077573.280297:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077573.280323:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff8b10037dd500 x1855007612720896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077573.280338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612720896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077573.280348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077573.280351:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077573.280355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612720896:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077573.280364:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077573.280393:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077573.280415:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff8b10037ddf80 x1855007612721024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077573.280432:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612721024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077573.280442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077573.280447:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077573.280454:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077573.280463:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077573.280490:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077573.280503:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff8b10037de300 x1855007612721152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077573.280518:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612721152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077573.280527:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077573.280535:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077573.280543:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077574.175818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077574.175897:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077575.199749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077575.199835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077576.223769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077576.223877:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077577.247769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077577.247851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077578.271780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077578.271854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077578.399781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.399790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077578.399794:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.399803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4566 00000100:00080000:1.0:1769077578.399813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.399818:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077578.399850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612721280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.399871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.399874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077578.399875:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.399879:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4566 00000100:00080000:1.0:1769077578.399881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.399885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077578.399892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077578.399902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612721408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077578.399908:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612721280:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077578.399912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.399919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077578.399920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.399923:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4566 00000100:00080000:1.0:1769077578.399926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.399930:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077578.399935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077578.399948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612721536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.399957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.399961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077578.399972:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077578.399977:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077578.399982:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4566 00000100:00080000:1.0:1769077578.399988:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.399993:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077578.399999:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077578.400012:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077578] req@ffff8b1006135880 x1855007612721280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077578.400015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612721664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077578.400035:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612721280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077578.400044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077578.400048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077578.400051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612721280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077578.400072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721408:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077578.400088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077578.400096:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077578] req@ffff8b1006137b80 x1855007612721408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077578.400105:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612721408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077578.400111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077578.400113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077578.400116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077578.400120:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077578.400132:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077578.400138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077578] req@ffff8b1006135c00 x1855007612721536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077578.400145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612721536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077578.400151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077578.400152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077578.400154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077578.400158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077578.400168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077578.400174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077578] req@ffff8b1006136a00 x1855007612721664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077578.400182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612721664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077578.400187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077578.400188:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077578.400190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077579.295777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077579.295778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077580.319749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077580.319812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077581.343742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077581.343818:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077582.367755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077582.367819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077583.391755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077583.391818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077583.519786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077583.519795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077583.519799:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077583.519807:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4571 00000100:00080000:3.0:1769077583.519818:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077583.519823:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077583.519855:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612721792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077583.519879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077583.519882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077583.519883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077583.519886:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4571 00000100:00080000:3.0:1769077583.519888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077583.519890:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077583.519897:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077583.519905:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612721920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077583.519913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077583.519915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077583.519916:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077583.519919:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4571 00000100:00080000:3.0:1769077583.519921:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077583.519923:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077583.519927:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077583.519932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612721792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077583.519936:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612722048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077583.519946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077583.519948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077583.519950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077583.519952:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4571 00000100:00080000:3.0:1769077583.519955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077583.519956:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077583.519959:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077583.519968:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612722176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077583.520000:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077583.520031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff8b10037ded80 x1855007612721792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077583.520051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612721792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077583.520073:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077583.520077:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077583.520082:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612721792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077583.520090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721920:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077583.520111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077583.520123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff8b10037de680 x1855007612721920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077583.520136:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612721920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077583.520146:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077583.520149:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077583.520153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612721920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077583.520173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077583.520189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077583.520199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff8b10037dc380 x1855007612722048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077583.520211:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612722048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077583.520220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077583.520223:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077583.520226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077583.520231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077583.520246:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077583.520255:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff8b10037dd180 x1855007612722176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077583.520267:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612722176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077583.520282:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077583.520286:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077583.520289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077584.415756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077584.415825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077585.439726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077585.439776:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077586.463820:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077586.463828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077587.487740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077587.487775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077588.511751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077588.511852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077588.639765:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077588.639779:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077588.639784:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077588.639794:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4576 00000100:00080000:3.0:1769077588.639807:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077588.639814:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077588.639861:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612722304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077588.639900:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077588.639905:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077588.639908:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077588.639912:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4576 00000100:00080000:3.0:1769077588.639915:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077588.639919:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077588.639928:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077588.639942:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612722432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077588.639951:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077588.639954:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077588.639956:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077588.639959:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4576 00000100:00080000:3.0:1769077588.639962:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077588.639964:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077588.639967:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077588.639976:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612722560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077588.639983:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077588.639986:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077588.639987:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077588.639989:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4576 00000100:00080000:3.0:1769077588.639991:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077588.639994:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077588.639997:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077588.640007:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612722688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077588.640014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612722304:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769077588.640083:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077588.640124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff8b10037dca80 x1855007612722304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077588.640144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612722304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077588.640156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077588.640159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077588.640162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612722304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077588.640169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722432:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077588.640188:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077588.640197:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff8b10037dd180 x1855007612722432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077588.640206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612722432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077588.640212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077588.640214:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077588.640216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077588.640220:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077588.640232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077588.640237:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff8b10037dc380 x1855007612722560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077588.640245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612722560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077588.640250:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077588.640252:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077588.640254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077588.640274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077588.640285:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077588.640291:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff8b10037de680 x1855007612722688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077588.640299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612722688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077588.640304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077588.640306:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077588.640308:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077589.535697:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077589.535808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077590.559774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077590.559868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077591.583747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077591.583803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077592.607752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077592.607827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077593.631800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077593.631835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077593.759758:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.759766:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077593.759770:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.759778:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4581 00000100:00080000:3.0:1769077593.759789:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.759795:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077593.759827:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612722816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.759855:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.759859:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077593.759861:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.759865:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4581 00000100:00080000:3.0:1769077593.759869:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.759873:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077593.759882:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077593.759894:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612722944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.759903:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077593.759908:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612722816:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077593.759908:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077593.759911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.759919:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4581 00000100:00080000:3.0:1769077593.759922:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.759924:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077593.759928:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077593.759937:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612723072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.759944:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.759946:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077593.759948:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.759950:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4581 00000100:00080000:3.0:1769077593.759960:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.759963:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077593.759966:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077593.759974:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077593.759976:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612723200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077593.760017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff8b10037ded80 x1855007612722816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077593.760038:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612722816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077593.760050:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077593.760055:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077593.760060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612722816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077593.760081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722944:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077593.760110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077593.760123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff8b10037de300 x1855007612722944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077593.760143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612722944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077593.760152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077593.760156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077593.760160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612722944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077593.760168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077593.760191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077593.760203:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff8b10037ddf80 x1855007612723072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077593.760215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612723072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077593.760223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077593.760226:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077593.760230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077593.760236:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077593.760254:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077593.760265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff8b10037dd500 x1855007612723200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077593.760277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612723200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077593.760285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077593.760288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077593.760291:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077594.655746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077594.655809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077595.679799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077595.679859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077596.703762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077596.703835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077597.727772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077597.727825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077598.751707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077598.751785:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077598.879794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077598.879806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077598.879811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077598.879822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4586 00000100:00080000:1.0:1769077598.879834:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077598.879841:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077598.879880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612723328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077598.879913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077598.879917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077598.879919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077598.879924:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4586 00000100:00080000:1.0:1769077598.879928:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077598.879932:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077598.879941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077598.879955:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612723456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077598.879967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077598.879973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077598.879975:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612723328:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077598.879976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077598.879986:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4586 00000100:00080000:1.0:1769077598.879991:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077598.879995:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077598.880001:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077598.880014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612723584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077598.880025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077598.880028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077598.880030:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077598.880034:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4586 00000100:00080000:1.0:1769077598.880039:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077598.880043:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077598.880050:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077598.880050:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077598.880068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612723712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077598.880097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff8b1006136300 x1855007612723328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077598.880119:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612723328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077598.880130:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077598.880134:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077598.880147:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612723328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077598.880172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077598.880195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077598.880209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff8b1006137800 x1855007612723456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077598.880223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612723456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077598.880232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077598.880236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077598.880239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077598.880246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077598.880271:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077598.880283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff8b1006135180 x1855007612723584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077598.880296:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612723584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077598.880305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077598.880309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077598.880313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077598.880335:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077598.880366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077598.880379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff8b1006136a00 x1855007612723712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077598.880393:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612723712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077598.880401:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077598.880404:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077598.880408:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077599.775746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077599.775803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077600.799805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077600.799818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077601.823713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077601.823828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077602.847708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077602.847771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077603.871749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077603.871816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077603.999724:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077603.999733:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077603.999737:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077603.999745:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4592 00000100:00080000:1.0:1769077603.999755:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077603.999760:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077603.999798:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612723840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077603.999829:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077603.999833:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077603.999835:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077603.999839:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4592 00000100:00080000:1.0:1769077603.999843:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077603.999847:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077603.999856:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077603.999879:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612723968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077603.999890:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077603.999896:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077603.999898:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077603.999904:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4592 00000100:00100000:2.0:1769077603.999904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612723840:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077603.999912:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077603.999918:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077603.999924:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077603.999937:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612724096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077603.999949:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077603.999953:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077603.999955:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077603.999959:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4592 00000100:00080000:1.0:1769077603.999963:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077603.999967:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077603.999972:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077603.999979:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077603.999993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612724224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077604.000004:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff8b1006135c00 x1855007612723840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.000026:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612723840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.000038:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.000043:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077604.000047:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612723840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077604.000055:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723968:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077604.000075:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077604.000085:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077604] req@ffff8b1006137b80 x1855007612723968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.000094:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612723968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.000100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.000102:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077604.000104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612723968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077604.000121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.000132:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077604.000138:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077604] req@ffff8b1006135880 x1855007612724096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.000151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612724096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.000156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.000157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077604.000159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077604.000163:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077604.000173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077604.000179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077604] req@ffff8b1006135500 x1855007612724224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077604.000186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612724224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077604.000191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077604.000193:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077604.000195:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077604.895707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077604.895786:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077605.919807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077605.919871:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077606.943730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077606.943820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077607.967821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077607.967825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077608.991702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077608.991788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077609.119757:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077609.119765:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077609.119769:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077609.119777:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4597 00000100:00080000:1.0:1769077609.119787:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077609.119793:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077609.119825:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612724352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077609.119849:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077609.119852:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077609.119853:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077609.119856:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4597 00000100:00080000:1.0:1769077609.119859:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077609.119861:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077609.119868:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077609.119876:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612724480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077609.119884:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077609.119886:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077609.119887:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077609.119890:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4597 00000100:00080000:1.0:1769077609.119892:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077609.119897:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077609.119897:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612724352:192.168.204.102@tcp:250:kworker.0 00000008:00100000:1.0:1769077609.119905:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077609.119913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612724608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077609.119920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077609.119922:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077609.119923:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077609.119926:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4597 00000100:00080000:1.0:1769077609.119929:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077609.119931:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077609.119934:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077609.119941:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612724736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077609.119957:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077609.119988:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff8b1006136d80 x1855007612724352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.120005:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612724352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.120014:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.120017:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077609.120021:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612724352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077609.120028:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077609.120046:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077609.120054:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff8b1006136680 x1855007612724480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.120064:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612724480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.120070:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.120072:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077609.120075:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077609.120095:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.120110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077609.120129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff8b1006137100 x1855007612724608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.120142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612724608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.120179:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.120181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077609.120184:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077609.120189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077609.120207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077609.120217:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff8b1006134e00 x1855007612724736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077609.120227:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612724736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077609.120234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077609.120237:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077609.120240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077610.015743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077610.015800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077611.039791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077611.039968:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077612.063760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077612.063855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077613.087776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077613.087896:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077614.111743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077614.111815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077614.239811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077614.239831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077614.239836:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077614.239846:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4602 00000100:00080000:3.0:1769077614.239858:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077614.239865:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077614.239902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612724864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077614.239927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077614.239929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077614.239931:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077614.239933:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4602 00000100:00080000:3.0:1769077614.239936:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077614.239940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077614.239947:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077614.239957:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612724992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077614.239966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077614.239967:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612724864:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077614.239973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077614.239976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077614.239979:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4602 00000100:00080000:3.0:1769077614.239981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077614.239984:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077614.239987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077614.239996:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612725120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077614.240003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077614.240005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077614.240006:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077614.240009:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4602 00000100:00080000:3.0:1769077614.240019:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077614.240021:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077614.240024:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077614.240029:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077614.240034:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612725248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077614.240088:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff8b10037dc000 x1855007612724864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077614.240103:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612724864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077614.240111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077614.240114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077614.240118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612724864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077614.240137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077614.240154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077614.240164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff8b10037dd880 x1855007612724992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077614.240172:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612724992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077614.240177:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077614.240180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077614.240182:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612724992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077614.240186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077614.240198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077614.240204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff8b10037df100 x1855007612725120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077614.240212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612725120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077614.240217:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077614.240219:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077614.240221:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077614.240224:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077614.240234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077614.240241:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff8b10037ddc00 x1855007612725248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077614.240249:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612725248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077614.240254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077614.240255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077614.240257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077615.135751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077615.135831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077616.159806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077616.159861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077617.183776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077617.183857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077618.207786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077618.207866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077619.231790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077619.231864:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077619.359804:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.359815:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077619.359820:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.359831:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4607 00000100:00080000:1.0:1769077619.359842:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.359849:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077619.359882:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612725376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.359906:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.359909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077619.359911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.359913:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4607 00000100:00080000:1.0:1769077619.359916:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.359919:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077619.359925:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077619.359932:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612725504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.359940:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.359945:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077619.359947:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.359950:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4607 00000100:00080000:1.0:1769077619.359952:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.359955:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077619.359958:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077619.359964:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612725376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077619.359969:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612725632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.360005:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.360007:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077619.360008:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.360011:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4607 00000100:00080000:1.0:1769077619.360015:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.360018:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077619.360022:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077619.360031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612725760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077619.360051:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077619.360088:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff8b1006134a80 x1855007612725376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077619.360107:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612725376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077619.360118:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077619.360122:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077619.360127:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612725376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077619.360135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725504:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077619.360159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077619.360172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff8b1006137480 x1855007612725504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077619.360192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612725504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077619.360201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077619.360204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077619.360207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077619.360230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077619.360251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077619.360262:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff8b1006134700 x1855007612725632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077619.360285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612725632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077619.360294:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077619.360297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077619.360301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077619.360307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077619.360326:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077619.360336:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff8b1006134e00 x1855007612725760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077619.360347:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612725760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077619.360360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077619.360363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077619.360366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612725760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077620.255816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077620.255882:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077621.279744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077621.279808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077622.303725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077622.303767:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077623.327788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077623.327848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077624.351764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077624.351868:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077624.479698:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077624.479706:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077624.479710:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077624.479718:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4612 00000100:00080000:1.0:1769077624.479728:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077624.479734:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077624.479765:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612725888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077624.479789:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077624.479792:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077624.479793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077624.479796:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4612 00000100:00080000:1.0:1769077624.479799:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077624.479801:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077624.479808:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077624.479817:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612726016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077624.479825:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077624.479828:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077624.479830:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077624.479832:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4612 00000100:00080000:1.0:1769077624.479834:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077624.479836:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077624.479840:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077624.479849:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612726144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077624.479856:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077624.479859:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077624.479862:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077624.479868:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4612 00000100:00100000:2.0:1769077624.479868:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612725888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077624.479872:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077624.479874:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077624.479877:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077624.479903:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612726272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077624.479997:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077624.480056:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff8b1006137100 x1855007612725888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077624.480074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612725888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077624.480084:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077624.480088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077624.480092:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612725888:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077624.480106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726016:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077624.480139:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077624.480159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff8b1006136680 x1855007612726016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077624.480171:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612726016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077624.480181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077624.480185:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077624.480189:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077624.480216:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077624.480234:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077624.480241:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff8b1006136d80 x1855007612726144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077624.480250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612726144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077624.480256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077624.480258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077624.480261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077624.480265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077624.480275:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077624.480281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff8b1006135500 x1855007612726272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077624.480289:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612726272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077624.480294:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077624.480296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077624.480298:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077625.375796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077625.375860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077626.399747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077626.399779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077627.423743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077627.423795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077628.447763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077628.447803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077629.471788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077629.471833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077629.599732:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077629.599744:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077629.599750:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077629.599760:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4617 00000100:00080000:3.0:1769077629.599771:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077629.599779:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077629.599838:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612726400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077629.599861:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077629.599864:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077629.599866:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077629.599869:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4617 00000100:00080000:3.0:1769077629.599872:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077629.599876:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077629.599883:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077629.599893:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612726528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077629.599901:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077629.599904:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077629.599905:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077629.599908:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4617 00000100:00080000:3.0:1769077629.599911:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077629.599912:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612726400:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077629.599916:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077629.599923:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077629.599932:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612726656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077629.599939:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077629.599941:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077629.599943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077629.599946:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4617 00000100:00080000:3.0:1769077629.599948:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077629.599950:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077629.599954:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077629.599961:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612726784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077629.600001:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077629.600033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077629.600055:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077630.495771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077630.495856:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077631.519710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077631.519823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077632.543748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077632.543826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077632.672016:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077632.672080:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077632.672105:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077632.672147:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077632.672157:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077632.672165:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769077632.672220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077632] req@ffff8b10037df800 x1855007612726656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077632.672251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612726656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077632.672264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077632.672270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077632.672274:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077632.672283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077632] req@ffff8b10037df480 x1855007612726784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077632.672295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612726784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077632.672305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077632.672309:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077632.672313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077632.672320:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077632] req@ffff8b10037dea00 x1855007612726400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077632.672335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612726400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077632.672344:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077632.672348:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077632.672352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612726400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077632.672360:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077632] req@ffff8b10037dc700 x1855007612726528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077632.672375:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612726528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077632.672383:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077632.672387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077632.672391:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612726528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077633.567763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077633.567830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077634.591776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077634.591847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077634.719808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077634.719818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077634.719821:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077634.719829:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4622 00000100:00080000:0.0:1769077634.719840:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077634.719845:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077634.719873:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612726912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077634.719895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077634.719898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077634.719899:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077634.719902:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4622 00000100:00080000:0.0:1769077634.719905:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077634.719907:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077634.719914:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077634.719922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612727040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077634.719929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077634.719932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077634.719934:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077634.719936:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4622 00000100:00080000:0.0:1769077634.719938:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077634.719941:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077634.719944:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077634.719956:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612726912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077634.719957:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612727168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077634.719973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077634.719977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077634.719979:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077634.719985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4622 00000100:00080000:0.0:1769077634.719991:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077634.719996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077634.720003:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077634.720015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612727296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077634.720031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077634.720085:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff8b1006026a00 x1855007612726912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077634.720106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612726912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077634.720123:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077634.720128:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077634.720132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612726912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077634.720140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077634.720164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077634.720177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff8b1006024000 x1855007612727040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077634.720189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612727040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077634.720198:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077634.720202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077634.720206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077634.720231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077634.720252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077634.720263:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff8b1006025880 x1855007612727168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077634.720276:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612727168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077634.720284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077634.720288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077634.720292:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077634.720298:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077634.720316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077634.720326:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff8b1006026300 x1855007612727296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077634.720338:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612727296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077634.720346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077634.720349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077634.720352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077635.615787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077635.615850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077636.639835:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077636.639870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077637.663791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077637.663872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077638.687756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077638.687860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077639.711779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077639.711876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077639.839777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077639.839789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077639.839794:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077639.839804:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4627 00000100:00080000:1.0:1769077639.839817:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077639.839824:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077639.839868:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612727424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077639.839898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077639.839901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077639.839903:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077639.839906:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4627 00000100:00080000:1.0:1769077639.839908:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077639.839911:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077639.839920:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077639.839933:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612727552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077639.839948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077639.839954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077639.839956:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077639.839961:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4627 00000100:00100000:2.0:1769077639.839962:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612727424:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077639.839964:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077639.839972:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077639.839975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077639.839985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612727680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077639.839991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077639.839995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077639.839996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077639.839998:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4627 00000100:00080000:1.0:1769077639.840001:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077639.840003:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077639.840006:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077639.840014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612727808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077639.840044:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077639.840079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff8b1006135880 x1855007612727424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.840095:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612727424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.840104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.840108:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077639.840111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612727424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077639.840118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077639.840134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077639.840142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff8b1006137b80 x1855007612727552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.840151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612727552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.840156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.840158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077639.840160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077639.840183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077639.840195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077639.840201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff8b1006135c00 x1855007612727680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.840209:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612727680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.840215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.840216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077639.840218:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077639.840222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077639.840233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077639.840239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff8b1006136a00 x1855007612727808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.840247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612727808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.840253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.840254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077639.840256:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612727808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077640.735749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077640.735840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077641.759715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077641.759776:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077642.783740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077642.783802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077643.807772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077643.807785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077644.831787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077644.831841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077644.959743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077644.959752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077644.959756:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077644.959764:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4633 00000100:00080000:1.0:1769077644.959774:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077644.959780:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077644.959813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612727936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077644.959838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077644.959842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077644.959844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077644.959848:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4633 00000100:00080000:1.0:1769077644.959852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077644.959855:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077644.959865:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077644.959877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612728064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077644.959887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077644.959890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077644.959892:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077644.959896:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4633 00000100:00100000:2.0:1769077644.959896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612727936:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077644.959899:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077644.959903:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077644.959907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077644.959916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612728192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077644.959925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077644.959927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077644.959929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077644.959932:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4633 00000100:00080000:1.0:1769077644.959936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077644.959939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077644.959944:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077644.959952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612728320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077644.959956:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077644.959989:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077644] req@ffff8b1006135180 x1855007612727936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077644.960007:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612727936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077644.960017:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077644.960021:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077644.960025:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612727936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077644.960032:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728064:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077644.960060:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077644.960077:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077644] req@ffff8b1006137800 x1855007612728064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077644.960091:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612728064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077644.960099:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077644.960103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077644.960107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077644.960143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077644.960173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077644.960190:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077644] req@ffff8b1006136300 x1855007612728192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077644.960202:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612728192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077644.960208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077644.960210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077644.960212:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077644.960217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077644.960233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077644.960242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077644] req@ffff8b1006136a00 x1855007612728320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077644.960257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612728320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077644.960272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077644.960275:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077644.960279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077645.855723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077645.855773:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077646.879725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077646.879789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077647.903729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077647.903798:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077648.927747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077648.927836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077649.951739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077649.951803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077650.079788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.079800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077650.079805:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.079816:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4638 00000100:00080000:0.0:1769077650.079827:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.079844:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077650.079882:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612728448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.079908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.079912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077650.079914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.079919:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4638 00000100:00080000:0.0:1769077650.079923:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.079929:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077650.079938:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077650.079945:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612728448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077650.079954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612728576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077650.079967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.079971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077650.079973:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.079978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4638 00000100:00080000:0.0:1769077650.079982:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.079987:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077650.079992:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077650.080004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612728704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077650.080015:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077650.080017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077650.080022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077650.080025:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077650.080031:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4638 00000100:00080000:0.0:1769077650.080036:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077650.080041:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077650.080047:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077650.080053:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff8b1006025180 x1855007612728448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077650.080061:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612728832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077650.080073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612728448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.080083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.080086:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077650.080090:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612728448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077650.080109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077650.080125:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077650.080134:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff8b1006027800 x1855007612728576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.080143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612728576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.080149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.080151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077650.080153:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077650.080157:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077650.080176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077650.080187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff8b1006027100 x1855007612728704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.080200:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612728704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.080208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.080211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077650.080215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077650.080232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077650.080251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077650.080260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff8b1006026680 x1855007612728832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.080268:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612728832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.080274:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.080276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077650.080279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612728832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077650.975803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077650.975827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077651.999792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077651.999864:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077653.023769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077653.023830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077654.047760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077654.047891:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077655.071771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077655.071808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077655.199755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077655.199767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077655.199772:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077655.199782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4643 00000100:00080000:0.0:1769077655.199795:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077655.199802:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077655.199844:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612728960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077655.199870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077655.199873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077655.199874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077655.199878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4643 00000100:00080000:0.0:1769077655.199880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077655.199884:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077655.199891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077655.199900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612729088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077655.199907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077655.199910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077655.199911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077655.199914:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4643 00000100:00080000:0.0:1769077655.199916:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077655.199918:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077655.199921:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077655.199929:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612729216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077655.199936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077655.199938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077655.199940:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077655.199942:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4643 00000100:00080000:0.0:1769077655.199943:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077655.199946:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077655.199949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077655.199954:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612729344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077655.199972:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612728960:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769077655.200059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077655.200103:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff8b1006026680 x1855007612728960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.200121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612728960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.200131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.200135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077655.200139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612728960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077655.200145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729088:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077655.200160:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077655.200168:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff8b1006027100 x1855007612729088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.200176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612729088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.200181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.200183:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077655.200187:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077655.200191:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077655.200204:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077655.200210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff8b1006027800 x1855007612729216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.200217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612729216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.200223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.200225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077655.200227:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077655.200253:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077655.200265:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077655.200271:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff8b1006025180 x1855007612729344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.200279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612729344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.200285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.200287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077655.200289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077656.095777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077656.095891:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077657.119744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077657.119819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077658.143855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077658.143877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077659.167811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077659.167845:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077660.191787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077660.191850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077660.319779:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.319796:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077660.319815:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.319829:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4648 00000100:00080000:0.0:1769077660.319844:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.319854:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077660.319929:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612729472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.319964:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.319977:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077660.319979:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.319985:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4648 00000100:00080000:0.0:1769077660.319990:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.319996:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077660.320009:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077660.320028:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612729472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077660.320029:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612729600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.320056:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.320059:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077660.320061:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.320065:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4648 00000100:00080000:0.0:1769077660.320069:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.320074:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077660.320079:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077660.320090:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612729728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.320098:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.320101:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077660.320104:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.320109:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4648 00000100:00080000:0.0:1769077660.320124:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.320129:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077660.320134:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077660.320141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077660.320151:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612729856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077660.320206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff8b1006026300 x1855007612729472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.320230:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612729472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.320243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.320249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077660.320253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612729472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077660.320283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729600:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077660.320332:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077660.320360:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff8b1006025880 x1855007612729600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.320370:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612729600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.320377:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.320379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077660.320382:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729600:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077660.320387:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077660.320410:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077660.320421:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff8b1006024000 x1855007612729728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.320435:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612729728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.320444:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.320448:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077660.320452:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077660.320460:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077660.320481:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077660.320494:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff8b1006026a00 x1855007612729856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.320507:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612729856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.320516:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.320520:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077660.320523:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612729856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077661.215760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077661.215876:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077662.239753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077662.239814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077663.263740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077663.263791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077664.287752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077664.287813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077665.311764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077665.311770:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077665.439819:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077665.439832:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077665.439836:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077665.439847:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4653 00000100:00080000:2.0:1769077665.439859:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077665.439866:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077665.439905:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612729984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077665.439936:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077665.439941:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077665.439943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077665.439947:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4653 00000100:00080000:2.0:1769077665.439951:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077665.439957:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077665.439967:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077665.439980:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612729984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077665.439993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612730112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077665.440007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077665.440011:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077665.440014:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077665.440019:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4653 00000100:00080000:2.0:1769077665.440023:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077665.440027:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077665.440033:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077665.440040:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077665.440050:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612730240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077665.440063:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077665.440067:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077665.440070:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077665.440074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff8b1005d59500 x1855007612729984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.440077:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4653 00000100:00080000:2.0:1769077665.440085:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077665.440092:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077665.440097:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612729984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077665.440098:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1769077665.440112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769077665.440117:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612730368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077665.440118:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077665.440124:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612729984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077665.440151:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077665.440169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077665.440178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff8b1005d59880 x1855007612730112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077665.440186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612730112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077665.440192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077665.440195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077665.440198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077665.440230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077665.440246:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077665.440254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff8b1005d5bb80 x1855007612730240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077665.440263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612730240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077665.440268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077665.440270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077665.440273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077665.440276:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077665.440287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077665.440294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff8b1005d5b480 x1855007612730368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077665.440302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612730368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077665.440307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077665.440308:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077665.440310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077666.335791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077666.335859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077667.359733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077667.359758:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077668.383757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077668.383819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077669.407758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077669.407818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077670.431734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077670.431803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077670.559755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077670.559764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077670.559767:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077670.559775:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4658 00000100:00080000:0.0:1769077670.559784:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077670.559791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077670.559823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612730496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077670.559846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077670.559849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077670.559850:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077670.559853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4658 00000100:00080000:0.0:1769077670.559855:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077670.559858:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077670.559864:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077670.559872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612730624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077670.559879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077670.559881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077670.559882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077670.559884:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4658 00000100:00080000:0.0:1769077670.559886:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077670.559888:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077670.559891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077670.559896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612730752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077670.559903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077670.559905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077670.559907:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077670.559910:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4658 00000100:00080000:0.0:1769077670.559913:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077670.559915:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077670.559918:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077670.559922:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612730496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077670.559927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612730880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077670.559994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077670.560025:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff8b1006026d80 x1855007612730496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077670.560039:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612730496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077670.560047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077670.560050:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077670.560053:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612730496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077670.560058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730624:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077670.560073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077670.560081:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff8b1006027b80 x1855007612730624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077670.560089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612730624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077670.560095:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077670.560097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077670.560099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077670.560102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077670.560113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077670.560137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff8b1006025f80 x1855007612730752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077670.560145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612730752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077670.560155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077670.560157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077670.560159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077670.560179:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077670.560192:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077670.560199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff8b1006024a80 x1855007612730880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077670.560206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612730880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077670.560211:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077670.560213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077670.560215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612730880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077671.455749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077671.455829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077672.479794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077672.479821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077673.503705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077673.503759:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077674.527742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077674.527763:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077675.551752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077675.551843:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077675.679749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077675.679758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077675.679762:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077675.679770:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4663 00000100:00080000:0.0:1769077675.679781:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077675.679786:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077675.679819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612731008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077675.679842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077675.679845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077675.679847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077675.679850:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4663 00000100:00080000:0.0:1769077675.679852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077675.679854:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077675.679862:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077675.679871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612731136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077675.679879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077675.679882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077675.679884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077675.679886:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612731008:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077675.679890:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4663 00000100:00080000:0.0:1769077675.679896:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077675.679899:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077675.679902:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077675.679912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612731264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077675.679918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077675.679920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077675.679921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077675.679925:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4663 00000100:00080000:0.0:1769077675.679939:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077675.679942:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077675.679961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077675.679966:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077675.679973:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612731392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077675.680003:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff8b1006027480 x1855007612731008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077675.680020:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612731008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077675.680044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077675.680048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077675.680051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612731008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077675.680088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077675.680106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077675.680116:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff8b1006024700 x1855007612731136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077675.680130:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612731136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077675.680139:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077675.680142:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077675.680154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077675.680161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077675.680182:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077675.680194:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff8b1006024e00 x1855007612731264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077675.680208:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612731264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077675.680216:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077675.680220:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077675.680223:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077675.680228:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077675.680259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077675.680288:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff8b1006024380 x1855007612731392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077675.680299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612731392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077675.680305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077675.680307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077675.680309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077676.575772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077676.575830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077677.599774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077677.599831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077678.623712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077678.623783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077679.647806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077679.647810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077680.671717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077680.671766:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077680.799726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077680.799735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077680.799739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077680.799748:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4668 00000100:00080000:0.0:1769077680.799758:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077680.799763:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077680.799793:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612731520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077680.799818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077680.799821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077680.799823:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077680.799827:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4668 00000100:00080000:0.0:1769077680.799830:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077680.799835:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077680.799838:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612731520:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769077680.799844:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077680.799857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612731648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077680.799865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077680.799868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077680.799870:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077680.799890:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4668 00000100:00080000:0.0:1769077680.799914:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077680.799919:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077680.799925:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077680.799927:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077680.799974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612731776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077680.799984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077680.799987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077680.799989:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077680.799994:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4668 00000100:00080000:0.0:1769077680.799997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077680.800001:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077680.800006:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077680.800006:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff8b1006024380 x1855007612731520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077680.800021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612731904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077680.800028:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612731520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077680.800039:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077680.800043:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077680.800046:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612731520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077680.800072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077680.800094:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077680.800107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff8b1006024e00 x1855007612731648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077680.800116:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612731648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077680.800126:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077680.800128:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077680.800131:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077680.800135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077680.800146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077680.800152:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff8b1006024700 x1855007612731776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077680.800160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612731776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077680.800165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077680.800167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077680.800169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077680.800172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077680.800182:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077680.800188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff8b1006027480 x1855007612731904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077680.800195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612731904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077680.800200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077680.800202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077680.800205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612731904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077681.695735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077681.695797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077682.719709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077682.719804:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077683.743761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077683.743859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077684.767706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077684.767770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077685.791754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077685.791902:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077685.919751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077685.919762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077685.919767:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077685.919778:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4674 00000100:00080000:1.0:1769077685.919790:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077685.919798:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077685.919842:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612732032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077685.919899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077685.919902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077685.919904:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077685.919908:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4674 00000100:00080000:1.0:1769077685.919912:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077685.919915:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077685.919925:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077685.919937:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612732160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077685.919946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077685.919952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077685.919954:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077685.919955:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612732032:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077685.919959:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4674 00000100:00080000:1.0:1769077685.919969:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077685.919974:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077685.919978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077685.919989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612732288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077685.919996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077685.919999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077685.920002:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077685.920005:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4674 00000100:00080000:1.0:1769077685.920009:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077685.920012:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077685.920015:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077685.920023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612732416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077685.920042:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077685.920075:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff8b1006135c00 x1855007612732032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.920089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612732032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.920098:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.920101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077685.920104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612732032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077685.920124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732160:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077685.920138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077685.920146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff8b1006137b80 x1855007612732160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.920154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612732160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.920160:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.920162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077685.920165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077685.920168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077685.920180:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077685.920186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff8b1006135880 x1855007612732288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.920194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612732288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.920199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.920201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077685.920203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077685.920207:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077685.920217:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077685.920223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff8b1006135500 x1855007612732416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.920231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612732416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.920236:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.920238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077685.920240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732416:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769077686.815811:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077686.815857:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077687.839742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077687.839813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077688.863745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077688.863794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077689.887745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077689.887819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077690.911855:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077690.911914:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077691.039801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077691.039813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077691.039818:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077691.039828:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4679 00000100:00080000:0.0:1769077691.039840:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077691.039846:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077691.039881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612732544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077691.039914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077691.039919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077691.039922:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077691.039926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4679 00000100:00080000:0.0:1769077691.039930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077691.039934:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077691.039942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077691.039953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612732672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077691.039962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077691.039964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077691.039966:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077691.039969:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4679 00000100:00080000:0.0:1769077691.039972:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077691.039974:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077691.039977:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077691.039981:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612732544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077691.039988:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612732800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077691.039996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077691.039999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077691.040000:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077691.040003:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4679 00000100:00080000:0.0:1769077691.040005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077691.040007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077691.040011:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077691.040021:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612732928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077691.040043:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077691.040073:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff8b1006024a80 x1855007612732544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.040087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612732544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.040095:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.040098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077691.040101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612732544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077691.040106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077691.040120:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077691.040127:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff8b1006025f80 x1855007612732672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.040135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612732672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.040141:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.040143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077691.040145:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077691.040163:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.040175:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077691.040181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff8b1006027b80 x1855007612732800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.040189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612732800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.040199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.040201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077691.040203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077691.040206:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077691.040216:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077691.040222:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077691/real 1769077691] req@ffff8b1006026d80 x1855007612732928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077691.040229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612732928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077691.040234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077691.040236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077691.040238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612732928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077691.935772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077691.935847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077692.959756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077692.959868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077693.983808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077693.983926:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077695.007788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077695.007874:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077696.031745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077696.031799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077696.159804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.159816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077696.159822:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.159836:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4684 00000100:00080000:0.0:1769077696.159850:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.159860:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077696.159910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612733056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.159939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.159942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077696.159944:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.159948:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4684 00000100:00080000:0.0:1769077696.159951:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.159955:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077696.159964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077696.159974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612733184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.159982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.159986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077696.159987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.159992:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4684 00000100:00080000:0.0:1769077696.159995:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.159998:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077696.160002:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077696.160009:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612733312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.160015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.160018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077696.160019:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.160022:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4684 00000100:00080000:0.0:1769077696.160024:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.160027:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077696.160029:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077696.160036:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612733440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077696.160052:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612733056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077696.160149:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077696.160169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077696.160194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077697.055775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077697.055839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077698.079741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077698.079794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077699.103745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077699.103825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077699.231985:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769077699.232047:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077699.232072:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077699.232124:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077699.232135:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769077699.232145:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769077699.232280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077699] req@ffff8b1006026300 x1855007612733440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077699.232311:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612733440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077699.232325:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077699.232332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077699.232338:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077699.232348:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077699] req@ffff8b1006026a00 x1855007612733056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077699.232364:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612733056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077699.232373:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077699.232377:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077699.232381:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612733056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077699.232389:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077699] req@ffff8b1006024000 x1855007612733184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077699.232403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612733184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077699.232413:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077699.232418:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077699.232422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077699.232429:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077699] req@ffff8b1006025880 x1855007612733312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077699.232443:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612733312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077699.232452:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077699.232456:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077699.232459:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077700.127722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077700.127808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077701.151838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077701.151842:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:1.0:1769077701.279758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.279770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077701.279775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.279785:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4689 00000100:00080000:1.0:1769077701.279796:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.279802:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077701.279835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612733568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.279867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.279871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077701.279874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.279879:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4689 00000100:00080000:1.0:1769077701.279883:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.279887:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077701.279898:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077701.279913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612733696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.279926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077701.279930:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612733568:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077701.279932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077701.279937:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.279944:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4689 00000100:00080000:1.0:1769077701.279949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.279954:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077701.279960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077701.279974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612733824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.279985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.279988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077701.279990:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.279997:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4689 00000800:00000100:2.0:1769077701.279997:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077701.280006:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.280011:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077701.280017:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077701.280032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612733952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077701.280040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff8b1006136d80 x1855007612733568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.280068:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612733568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.280076:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.280080:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077701.280083:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612733568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077701.280106:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077701.280130:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077701.280145:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff8b1006136680 x1855007612733696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.280160:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612733696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.280169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.280173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077701.280176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077701.280202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077701.280227:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077701.280239:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff8b1006137100 x1855007612733824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.280252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612733824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.280261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.280264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077701.280268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077701.280275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077701.280308:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077701.280321:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff8b1006134e00 x1855007612733952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.280335:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612733952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.280343:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.280346:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077701.280349:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612733952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077702.175738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077702.175795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077703.199756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077703.199814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077704.223706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077704.223788:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077705.247721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077705.247782:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077706.271778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077706.271782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077706.399826:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.399838:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077706.399843:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.399854:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4694 00000100:00080000:0.0:1769077706.399867:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.399875:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077706.399919:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612734080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.399946:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.399949:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077706.399951:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.399954:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4694 00000100:00080000:0.0:1769077706.399957:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.399959:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077706.399968:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077706.399977:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612734208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.399985:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.399988:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077706.399989:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.399992:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4694 00000100:00080000:0.0:1769077706.399994:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.399997:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077706.400000:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077706.400007:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612734336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.400013:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.400016:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077706.400018:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.400021:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4694 00000100:00080000:0.0:1769077706.400024:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.400026:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077706.400031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077706.400043:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612734464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077706.400044:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612734080:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769077706.400132:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077706.400167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff8b1006025180 x1855007612734080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.400185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612734080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.400197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.400202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077706.400207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612734080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077706.400215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077706.400238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077706.400248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff8b1006027800 x1855007612734208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.400257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612734208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.400264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.400266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077706.400269:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077706.400274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077706.400288:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077706.400295:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff8b1006027100 x1855007612734336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.400308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612734336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.400316:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.400319:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077706.400323:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077706.400352:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077706.400371:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077706.400379:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff8b1006026680 x1855007612734464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.400387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612734464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.400392:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.400395:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077706.400397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077707.295784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077707.295841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077708.319750:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077708.319827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077709.343763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077709.343820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077710.367729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077710.367782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077711.391726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077711.391776:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077711.519732:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077711.519742:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077711.519747:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077711.519756:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4699 00000100:00080000:0.0:1769077711.519765:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077711.519770:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077711.519800:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612734592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077711.519828:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077711.519830:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077711.519832:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077711.519834:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4699 00000100:00080000:0.0:1769077711.519836:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077711.519839:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077711.519845:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077711.519852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612734720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077711.519859:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077711.519862:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077711.519863:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077711.519865:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4699 00000100:00080000:0.0:1769077711.519868:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077711.519870:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077711.519873:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077711.519879:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612734848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077711.519888:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612734592:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077711.519892:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077711.519898:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077711.519900:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077711.519904:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4699 00000100:00080000:0.0:1769077711.519907:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077711.519911:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077711.519916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077711.519929:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612734976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077711.519953:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077711.519988:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077711] req@ffff8b1006026680 x1855007612734592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077711.520006:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612734592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077711.520016:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077711.520020:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077711.520025:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612734592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077711.520032:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734720:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077711.520054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077711.520066:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077711] req@ffff8b1006027100 x1855007612734720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077711.520077:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612734720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077711.520093:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077711.520097:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077711.520101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077711.520125:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077711.520143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077711.520154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077711] req@ffff8b1006027800 x1855007612734848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077711.520166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612734848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077711.520174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077711.520177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077711.520180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077711.520186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077711.520202:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077711.520212:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077711] req@ffff8b1006025180 x1855007612734976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077711.520222:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612734976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077711.520229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077711.520232:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077711.520235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612734976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077712.415739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077712.415815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077713.439768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077713.439824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077714.463748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077714.463849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077715.487854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077715.487876:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077716.511755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077716.511774:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077716.639746:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077716.639758:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077716.639762:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077716.639772:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4704 00000100:00080000:0.0:1769077716.639782:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077716.639789:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077716.639825:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612735104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077716.639847:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077716.639851:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077716.639853:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077716.639856:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4704 00000100:00080000:0.0:1769077716.639859:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077716.639865:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077716.639873:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077716.639886:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612735232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077716.639889:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612735104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077716.639898:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077716.639901:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077716.639903:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077716.639905:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4704 00000100:00080000:0.0:1769077716.639908:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077716.639910:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077716.639914:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077716.639927:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612735360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077716.639934:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077716.639936:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077716.639937:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077716.639940:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4704 00000100:00080000:0.0:1769077716.639949:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077716.639952:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077716.639955:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077716.639960:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077716.639970:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612735488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077716.640015:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff8b1006025880 x1855007612735104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.640041:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612735104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.640051:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.640056:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077716.640059:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612735104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077716.640088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077716.640110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077716.640119:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff8b1006024000 x1855007612735232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.640127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612735232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.640133:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.640135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077716.640137:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077716.640141:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077716.640154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077716.640160:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff8b1006026a00 x1855007612735360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.640168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612735360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.640174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.640176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077716.640180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077716.640185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077716.640212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077716.640225:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff8b1006026300 x1855007612735488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.640238:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612735488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.640246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.640249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077716.640253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077717.535754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077717.535823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077718.559764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077718.559826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077719.583783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077719.583856:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077720.607745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077720.607799:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077721.631735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077721.631779:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077721.759788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077721.759800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077721.759805:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077721.759822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4709 00000100:00080000:1.0:1769077721.759836:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077721.759843:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077721.759886:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612735616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077721.759916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077721.759920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077721.759921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077721.759926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4709 00000100:00080000:1.0:1769077721.759930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077721.759934:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077721.759950:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077721.759962:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612735744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077721.759979:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612735616:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077721.759981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077721.759993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077721.759996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077721.760000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4709 00000100:00080000:1.0:1769077721.760005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077721.760010:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077721.760016:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077721.760030:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612735872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077721.760039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077721.760043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077721.760045:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077721.760049:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4709 00000100:00080000:1.0:1769077721.760054:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077721.760061:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769077721.760063:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769077721.760070:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077721.760087:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612736000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077721.760110:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff8b1006134700 x1855007612735616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.760144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612735616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.760155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.760160:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077721.760164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612735616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077721.760186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735744:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077721.760209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077721.760221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff8b1006137480 x1855007612735744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.760234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612735744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.760242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.760246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077721.760249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077721.760279:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077721.760301:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077721.760312:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff8b1006134a80 x1855007612735872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.760331:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612735872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.760338:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.760342:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077721.760345:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612735872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077721.760351:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077721.760369:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077721.760380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff8b1006134e00 x1855007612736000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.760392:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612736000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.760400:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.760403:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077721.760406:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077722.655788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077722.655996:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077723.679723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077723.679790:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077724.703753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077724.703804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077725.727722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077725.727760:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077726.751754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077726.751826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077726.879706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077726.879717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077726.879721:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077726.879730:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4714 00000100:00080000:1.0:1769077726.879740:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077726.879747:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077726.879781:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612736128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077726.879809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077726.879813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077726.879815:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077726.879819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4714 00000100:00080000:1.0:1769077726.879822:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077726.879826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077726.879835:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077726.879847:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612736256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077726.879858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077726.879863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1769077726.879865:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612736128:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077726.879866:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077726.879874:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4714 00000100:00080000:1.0:1769077726.879879:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077726.879883:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077726.879888:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077726.879899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612736384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077726.879908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077726.879912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077726.879914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077726.879918:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4714 00000100:00080000:1.0:1769077726.879922:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077726.879925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077726.879930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077726.879932:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077726.879946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612736512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077726.879971:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff8b1006137100 x1855007612736128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.879991:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612736128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.880001:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.880006:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077726.880009:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612736128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077726.880030:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077726.880063:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077726.880076:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff8b1006136680 x1855007612736256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.880089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612736256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.880098:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.880101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077726.880104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077726.880109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077726.880127:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077726.880137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff8b1006136d80 x1855007612736384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.880149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612736384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.880163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.880193:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077726.880197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077726.880203:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077726.880223:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077726.880234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff8b1006135500 x1855007612736512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.880245:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612736512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.880253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.880256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077726.880259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077727.775705:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077727.775754:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077728.799735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077728.799779:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077729.823813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077729.823833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077730.847715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077730.847772:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077731.871702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077731.871764:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077731.999756:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.999764:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077731.999767:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.999777:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4720 00000100:00080000:2.0:1769077731.999787:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.999793:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077731.999830:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612736640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.999854:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.999857:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077731.999858:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.999861:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4720 00000100:00080000:2.0:1769077731.999864:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.999868:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077731.999874:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077731.999882:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612736640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077731.999886:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612736768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.999894:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.999897:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077731.999898:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.999900:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4720 00000100:00080000:2.0:1769077731.999902:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.999904:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077731.999907:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077731.999914:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612736896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.999920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.999922:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077731.999923:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.999926:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4720 00000100:00080000:2.0:1769077731.999929:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.999931:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077731.999934:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077731.999936:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077731.999943:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612737024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077731.999967:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff8b1005d58e00 x1855007612736640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077731.999980:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612736640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077731.999987:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077731.999989:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077731.999992:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612736640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077732.000009:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077732.000022:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077732.000029:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077732] req@ffff8b1005d5a300 x1855007612736768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077732.000036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612736768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077732.000041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077732.000043:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077732.000044:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077732.000048:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077732.000059:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077732.000064:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077732] req@ffff8b1005d58000 x1855007612736896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077732.000071:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612736896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077732.000075:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077732.000076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077732.000078:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612736896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077732.000102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077732.000115:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077732.000122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077732] req@ffff8b1005d58a80 x1855007612737024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077732.000129:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612737024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077732.000134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077732.000136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077732.000138:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077732.895711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077732.895753:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077733.919734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077733.919791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077734.943724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077734.943775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077735.967738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077735.967792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077736.991807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077736.992053:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077737.119792:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.119804:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077737.119808:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.119818:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4725 00000100:00080000:0.0:1769077737.119829:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.119835:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077737.119873:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612737152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.119905:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.119909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.119911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.119916:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4725 00000100:00080000:0.0:1769077737.119919:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.119923:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077737.119932:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.119946:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612737280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077737.119947:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612737152:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077737.119961:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.119964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077737.119966:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077737.119969:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4725 00000100:00080000:0.0:1769077737.119972:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.119974:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077737.119978:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.119986:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612737408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.119992:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077737.120022:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077737.120024:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077737.120029:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077737.120032:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4725 00000100:00080000:0.0:1769077737.120036:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077737.120040:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077737.120044:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.120054:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612737536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077737.120068:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff8b1006026d80 x1855007612737152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077737.120089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612737152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077737.120098:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077737.120101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077737.120104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612737152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077737.120122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737280:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077737.120139:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077737.120148:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff8b1006027b80 x1855007612737280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077737.120156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612737280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077737.120162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077737.120164:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077737.120166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077737.120170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077737.120181:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077737.120187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff8b1006025f80 x1855007612737408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077737.120194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612737408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077737.120204:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077737.120206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077737.120208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077737.120212:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077737.120222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077737.120228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff8b1006024a80 x1855007612737536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077737.120235:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612737536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077737.120248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077737.120251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077737.120253:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077738.015754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077738.015816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077739.039763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077739.039785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077740.063759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077740.063922:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077741.087757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077741.087845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077742.111760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077742.111828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077742.239791:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077742.239800:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077742.239804:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077742.239827:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4730 00000100:00080000:2.0:1769077742.239838:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077742.239844:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077742.239880:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612737664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077742.239920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077742.239925:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077742.239927:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077742.239947:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4730 00000100:00080000:2.0:1769077742.239950:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077742.239954:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077742.239961:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077742.239971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612737792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077742.239982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612737664:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077742.239985:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077742.239997:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077742.239998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077742.240002:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4730 00000100:00080000:2.0:1769077742.240004:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077742.240006:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077742.240011:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077742.240023:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612737920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077742.240029:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077742.240032:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077742.240034:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077742.240036:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4730 00000100:00080000:2.0:1769077742.240038:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077742.240040:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077742.240044:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077742.240051:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612738048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077742.240074:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077742.240116:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff8b1005d59180 x1855007612737664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077742.240132:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612737664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077742.240142:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077742.240146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077742.240149:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612737664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077742.240176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077742.240193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077742.240201:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff8b1005d5b800 x1855007612737792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077742.240210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612737792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077742.240216:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077742.240218:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077742.240220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077742.240224:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077742.240244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077742.240257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff8b1005d58a80 x1855007612737920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077742.240271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612737920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077742.240281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077742.240284:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077742.240288:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612737920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077742.240295:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077742.240319:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077742.240332:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff8b1005d58000 x1855007612738048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077742.240346:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612738048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077742.240356:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077742.240359:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077742.240363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077743.135723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077743.135794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077744.159797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077744.159842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077745.183731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077745.183767:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077746.207760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077746.207871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077747.231759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077747.231824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077747.359788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077747.359797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077747.359800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077747.359808:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4735 00000100:00080000:1.0:1769077747.359818:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077747.359848:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077747.359881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612738176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077747.359906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077747.359909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077747.359910:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077747.359913:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4735 00000100:00080000:1.0:1769077747.359916:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077747.359919:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077747.359927:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077747.359936:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612738304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077747.359940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612738176:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077747.359956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077747.359959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077747.359960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077747.359964:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4735 00000100:00080000:1.0:1769077747.359967:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077747.359969:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077747.359973:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077747.359981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612738432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077747.359987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077747.359990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077747.359991:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077747.359994:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4735 00000100:00080000:1.0:1769077747.359997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077747.359999:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077747.360002:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077747.360009:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077747.360011:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612738560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077747.360053:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff8b1006135880 x1855007612738176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077747.360070:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612738176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077747.360078:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077747.360082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077747.360085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612738176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077747.360103:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738304:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077747.360120:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077747.360129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff8b1006137b80 x1855007612738304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077747.360137:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612738304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077747.360143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077747.360146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077747.360148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077747.360152:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077747.360163:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077747.360169:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff8b1006135c00 x1855007612738432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077747.360177:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612738432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077747.360182:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077747.360184:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077747.360186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077747.360190:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077747.360200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077747.360206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff8b1006136a00 x1855007612738560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077747.360224:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612738560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077747.360233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077747.360235:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077747.360238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077748.255738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077748.255802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077749.279743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077749.279823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077750.303716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077750.303771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077751.327821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077751.327878:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077752.351751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077752.351814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077752.479760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.479769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077752.479773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.479781:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4740 00000100:00080000:1.0:1769077752.479793:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.479801:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077752.479839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612738688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077752.479877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.479881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077752.479883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.479887:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4740 00000100:00080000:1.0:1769077752.479891:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.479896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077752.479906:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077752.479923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612738816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077752.479935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077752.479936:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612738688:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077752.479945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077752.479949:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.479954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4740 00000100:00080000:1.0:1769077752.479959:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.479964:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077752.479970:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077752.479982:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612738944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077752.479993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.479997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769077752.480002:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077752.480003:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.480010:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4740 00000100:00080000:1.0:1769077752.480016:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.480021:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077752.480026:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077752.480040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff8b1006136300 x1855007612738688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077752.480044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612739072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077752.480067:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612738688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077752.480076:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077752.480081:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077752.480085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612738688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077752.480105:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738816:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077752.480124:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077752.480137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff8b1006137800 x1855007612738816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077752.480150:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612738816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077752.480159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077752.480163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077752.480166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077752.480173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077752.480195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077752.480206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff8b1006135180 x1855007612738944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077752.480215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612738944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077752.480220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077752.480222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077752.480224:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612738944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077752.480228:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077752.480242:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077752.480249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff8b1006136a00 x1855007612739072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077752.480258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612739072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077752.480264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077752.480266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077752.480268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077753.375754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077753.375845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077754.399773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077754.399833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077755.423755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077755.423812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077756.447813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077756.447863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077757.471797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077757.471856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077757.599734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.599743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077757.599747:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.599755:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4745 00000100:00080000:1.0:1769077757.599765:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.599771:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077757.599809:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612739200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.599838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.599842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077757.599843:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.599846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4745 00000100:00080000:1.0:1769077757.599850:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.599853:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077757.599861:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077757.599877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612739328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077757.599879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612739200:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077757.599902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.599905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077757.599906:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.599909:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4745 00000100:00080000:1.0:1769077757.599912:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.599914:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077757.599918:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077757.599926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612739456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.599933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.599936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077757.599937:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.599940:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4745 00000100:00080000:1.0:1769077757.599942:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.599944:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077757.599950:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077757.599950:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077757.599961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612739584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077757.599987:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff8b1006135c00 x1855007612739200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077757.600008:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612739200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077757.600020:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077757.600025:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077757.600029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612739200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077757.600050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077757.600073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077757.600085:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff8b1006137b80 x1855007612739328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077757.600099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612739328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077757.600108:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077757.600112:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077757.600115:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077757.600122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077757.600144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077757.600155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff8b1006135880 x1855007612739456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077757.600176:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612739456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077757.600184:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077757.600188:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077757.600192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077757.600198:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077757.600215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077757.600224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff8b1006135500 x1855007612739584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077757.600236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612739584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077757.600245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077757.600249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077757.600252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077758.495781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077758.495794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077759.519741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077759.519827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077760.543720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077760.543772:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077761.567730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077761.567791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077762.591735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077762.591778:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077762.719788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.719801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077762.719806:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.719817:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4750 00000100:00080000:3.0:1769077762.719830:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.719837:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077762.719888:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612739712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077762.719917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.719921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077762.719923:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.719928:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4750 00000100:00080000:3.0:1769077762.719932:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.719937:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077762.719949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077762.719966:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612739840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077762.719979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.719984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077762.719986:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.719991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4750 00000100:00080000:3.0:1769077762.719995:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.720000:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077762.720006:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077762.720020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612739968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077762.720023:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612739712:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077762.720030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.720047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077762.720049:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.720054:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4750 00000100:00080000:3.0:1769077762.720057:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.720061:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077762.720066:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077762.720076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612740096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077762.720142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077762.720183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077762.720204:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077763.615750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077763.615819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077764.639708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077764.639761:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077765.663800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077765.663801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000400:00000100:1.0:1769077765.791944:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769077765.791989:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769077765.792010:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077765.792054:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077765.792065:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077765.792073:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769077765.792815:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077765] req@ffff8b10037dea00 x1855007612739968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077765.792842:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612739968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077765.792855:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077765.792860:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077765.792865:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077765.792873:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077765] req@ffff8b10037df480 x1855007612740096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077765.792884:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612740096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077765.792892:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077765.792897:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077765.792900:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077765.792907:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077765] req@ffff8b10037dce00 x1855007612739712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077765.792927:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612739712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077765.792934:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077765.792938:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077765.792942:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612739712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077765.792949:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077765] req@ffff8b10037dc700 x1855007612739840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077765.792960:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612739840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077765.792969:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077765.792972:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077765.792975:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612739840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077766.687696:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077766.687771:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077767.711746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077767.711812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077767.839729:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077767.839738:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077767.839741:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077767.839750:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4755 00000100:00080000:0.0:1769077767.839760:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077767.839766:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077767.839798:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612740224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077767.839826:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077767.839830:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077767.839833:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077767.839838:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4755 00000100:00080000:0.0:1769077767.839842:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077767.839847:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077767.839866:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077767.839874:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612740224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077767.839886:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612740352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077767.839900:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077767.839904:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077767.839907:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077767.839911:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4755 00000100:00080000:0.0:1769077767.839917:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077767.839922:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077767.839927:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077767.839944:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612740480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077767.839944:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077767.839960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077767.839964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077767.839966:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077767.839970:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4755 00000100:00080000:0.0:1769077767.839975:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077767.839979:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077767.839985:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff8b1006027480 x1855007612740224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077767.839986:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077767.840005:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612740608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077767.840013:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612740224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077767.840029:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077767.840035:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077767.840041:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612740224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077767.840066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077767.840088:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077767.840100:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff8b1006024700 x1855007612740352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077767.840115:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612740352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077767.840123:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077767.840126:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077767.840130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077767.840137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077767.840157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077767.840169:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff8b1006024e00 x1855007612740480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077767.840182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612740480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077767.840191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077767.840195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077767.840198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077767.840204:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077767.840224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077767.840242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff8b1006024380 x1855007612740608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077767.840254:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612740608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077767.840263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077767.840266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077767.840270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077768.735743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077768.735812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077769.759736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077769.759859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077770.783744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077770.783808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077771.807752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077771.807808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077772.831802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077772.831820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077772.959814:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077772.959825:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077772.959831:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077772.959843:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4761 00000100:00080000:2.0:1769077772.959855:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077772.959862:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077772.959899:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612740736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077772.959929:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077772.959933:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077772.959935:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077772.959940:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4761 00000100:00080000:2.0:1769077772.959944:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077772.959947:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077772.959957:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077772.959971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612740864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077772.959982:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077772.959987:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769077772.959989:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612740736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077772.959990:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077772.960001:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4761 00000100:00080000:2.0:1769077772.960005:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077772.960008:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077772.960014:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077772.960027:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612740992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077772.960037:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077772.960050:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077772.960053:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077772.960057:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4761 00000100:00080000:2.0:1769077772.960061:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077772.960067:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769077772.960068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:2.0:1769077772.960076:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077772.960092:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612741120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077772.960114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff8b1005d5a300 x1855007612740736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077772.960135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612740736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077772.960146:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077772.960150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077772.960155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612740736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077772.960180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077772.960204:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077772.960219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff8b1005d58e00 x1855007612740864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077772.960233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612740864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077772.960243:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077772.960246:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077772.960250:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077772.960257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077772.960278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077772.960291:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff8b1005d5b480 x1855007612740992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077772.960304:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612740992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077772.960314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077772.960317:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077772.960320:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612740992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077772.960326:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077772.960347:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077772.960360:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff8b1005d5bb80 x1855007612741120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077772.960373:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612741120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077772.960381:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077772.960385:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077772.960389:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077773.855792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077773.855864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077774.879766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077774.879799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077775.903799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077775.903871:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077776.927787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077776.927849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077777.951724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077777.951776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077778.079755:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077778.079765:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077778.079769:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077778.079778:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4766 00000100:00080000:2.0:1769077778.079795:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077778.079801:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077778.079831:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612741248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077778.079858:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077778.079862:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077778.079864:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077778.079869:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4766 00000100:00080000:2.0:1769077778.079873:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077778.079878:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077778.079887:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077778.079903:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612741376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077778.079908:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612741248:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077778.079920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077778.079923:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077778.079925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077778.079929:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4766 00000100:00080000:2.0:1769077778.079932:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077778.079935:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077778.079940:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077778.079950:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612741504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077778.079957:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077778.079960:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077778.079961:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077778.079964:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4766 00000100:00080000:2.0:1769077778.079966:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077778.079968:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077778.079971:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077778.079974:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077778.079983:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612741632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077778.080008:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff8b1005d59880 x1855007612741248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077778.080023:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612741248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077778.080031:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077778.080034:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077778.080037:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612741248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077778.080056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077778.080070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077778.080079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff8b1005d59500 x1855007612741376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077778.080087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612741376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077778.080092:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077778.080095:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077778.080097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077778.080101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077778.080112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077778.080118:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff8b1005d5aa00 x1855007612741504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077778.080125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612741504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077778.080131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077778.080133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077778.080135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077778.080142:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077778.080161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077778.080174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077778] req@ffff8b1005d5b100 x1855007612741632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077778.080188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612741632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077778.080198:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077778.080202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077778.080205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077778.975764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077778.975827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077779.999764:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077779.999777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077781.023788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077781.023880:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077782.047737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077782.047814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077783.071749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077783.071804:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077783.199701:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077783.199710:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077783.199714:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077783.199723:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4771 00000100:00080000:2.0:1769077783.199733:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077783.199739:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077783.199771:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612741760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077783.199794:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077783.199797:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077783.199798:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077783.199801:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4771 00000100:00080000:2.0:1769077783.199803:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077783.199805:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077783.199812:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077783.199821:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612741888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077783.199829:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077783.199832:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077783.199833:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077783.199838:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4771 00000100:00100000:1.0:1769077783.199840:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612741760:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077783.199840:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077783.199845:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077783.199851:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077783.199861:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612742016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077783.199867:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077783.199870:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077783.199872:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077783.199874:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4771 00000100:00080000:2.0:1769077783.199877:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077783.199879:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077783.199882:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077783.199888:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612742144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077783.199903:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077783.199934:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff8b1005d58380 x1855007612741760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077783.199948:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612741760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077783.199956:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077783.199959:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077783.199962:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612741760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077783.199980:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077783.199994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077783.200002:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff8b1005d5ad80 x1855007612741888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077783.200011:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612741888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077783.200017:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077783.200019:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077783.200021:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612741888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077783.200025:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077783.200036:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077783.200041:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff8b1005d5b100 x1855007612742016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077783.200058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612742016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077783.200064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077783.200066:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077783.200068:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077783.200072:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077783.200084:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077783.200090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff8b1005d5aa00 x1855007612742144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077783.200098:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612742144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077783.200107:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077783.200110:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077783.200112:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077784.095702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077784.095751:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077785.119751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077785.119821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077786.143750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077786.143837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077787.167785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077787.167813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077788.191755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077788.191843:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077788.319773:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077788.319785:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077788.319790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077788.319802:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4776 00000100:00080000:2.0:1769077788.319813:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077788.319821:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077788.319865:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612742272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077788.319906:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077788.319911:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077788.319913:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077788.319918:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4776 00000100:00080000:2.0:1769077788.319923:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077788.319927:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077788.319939:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077788.319959:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612742400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077788.319959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612742272:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077788.319973:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077788.319978:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077788.319980:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077788.319986:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4776 00000100:00080000:2.0:1769077788.319991:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077788.319996:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077788.320002:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077788.320015:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612742528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077788.320024:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077788.320027:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1769077788.320030:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769077788.320032:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077788.320038:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4776 00000100:00080000:2.0:1769077788.320043:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077788.320048:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077788.320053:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077788.320065:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612742656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077788.320125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff8b1005d59500 x1855007612742272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.320252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612742272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.320261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.320265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077788.320267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612742272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077788.320291:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742400:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077788.320314:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077788.320327:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff8b1005d59880 x1855007612742400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.320379:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612742400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.320388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.320392:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077788.320395:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077788.320403:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077788.320427:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077788.320440:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff8b1005d5bb80 x1855007612742528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.320448:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612742528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.320459:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.320461:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077788.320463:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077788.320468:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077788.320481:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077788.320487:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff8b1005d5b480 x1855007612742656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.320495:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612742656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.320509:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.320511:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077788.320513:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077789.215751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077789.215847:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077790.239741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077790.239799:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077791.263750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077791.263801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077792.287755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077792.287818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077793.311759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077793.311853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077793.439798:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077793.439806:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077793.439810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077793.439818:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4781 00000100:00080000:3.0:1769077793.439827:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077793.439832:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077793.439864:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612742784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077793.439891:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077793.439894:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077793.439895:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077793.439898:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4781 00000100:00080000:3.0:1769077793.439900:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077793.439902:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077793.439909:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077793.439917:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612742912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077793.439924:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077793.439927:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077793.439928:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077793.439930:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4781 00000100:00080000:3.0:1769077793.439931:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077793.439934:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077793.439936:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077793.439941:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612743040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077793.439948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077793.439950:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077793.439951:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077793.439954:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4781 00000100:00080000:3.0:1769077793.439957:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077793.439960:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077793.439963:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077793.439972:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612743168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077793.439973:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612742784:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769077793.440035:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077793.440090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff8b10037df800 x1855007612742784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077793.440106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612742784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077793.440113:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077793.440123:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077793.440126:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612742784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077793.440135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077793.440151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077793.440159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff8b10037ddc00 x1855007612742912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077793.440167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612742912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077793.440173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077793.440175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077793.440177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612742912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077793.440182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077793.440193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077793.440199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff8b10037df100 x1855007612743040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077793.440206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612743040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077793.440212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077793.440213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077793.440215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077793.440232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077793.440242:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077793.440249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff8b10037dd880 x1855007612743168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077793.440256:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612743168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077793.440261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077793.440263:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077793.440265:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077794.335811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077794.335852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077795.359721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077795.359797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077796.383746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077796.383806:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077797.407716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077797.407789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077798.431792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077798.431862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077798.559747:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077798.559759:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077798.559764:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077798.559774:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4786 00000100:00080000:3.0:1769077798.559786:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077798.559794:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077798.559829:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612743296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077798.559862:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077798.559866:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077798.559868:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077798.559873:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4786 00000100:00080000:3.0:1769077798.559877:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077798.559881:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077798.559891:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077798.559906:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612743424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077798.559920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612743296:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077798.559920:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077798.559931:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077798.559935:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077798.559940:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4786 00000100:00080000:3.0:1769077798.559945:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077798.559949:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077798.559956:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077798.559989:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612743552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077798.560001:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769077798.560004:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077798.560007:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077798.560011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077798.560017:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4786 00000100:00080000:3.0:1769077798.560023:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077798.560028:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077798.560034:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077798.560048:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff8b10037dc000 x1855007612743296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.560050:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612743680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077798.560087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612743296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077798.560100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077798.560106:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077798.560112:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612743296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077798.560135:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077798.560158:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077798.560171:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff8b10037dd500 x1855007612743424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077798.560186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612743424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077798.560195:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077798.560199:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077798.560203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077798.560210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077798.560229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077798.560240:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff8b10037ddf80 x1855007612743552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077798.560254:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612743552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077798.560270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077798.560274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077798.560278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077798.560284:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077798.560302:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077798.560312:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff8b10037de300 x1855007612743680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077798.560325:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612743680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077798.560334:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077798.560338:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077798.560342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077799.455714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077799.455802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077800.479750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077800.479847:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077801.503755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077801.503781:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077802.527832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077802.528003:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077803.551770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077803.551955:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077803.679775:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077803.679784:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077803.679789:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077803.679798:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4791 00000100:00080000:3.0:1769077803.679808:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077803.679814:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077803.679846:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612743808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077803.679865:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077803.679868:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077803.679869:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077803.679873:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4791 00000100:00080000:3.0:1769077803.679889:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077803.679893:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077803.679902:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077803.679913:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612743808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077803.679915:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612743936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077803.679927:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077803.679929:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077803.679931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077803.679934:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4791 00000100:00080000:3.0:1769077803.679937:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077803.679939:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077803.679943:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077803.679951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612744064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077803.679961:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077803.679964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077803.679966:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077803.679968:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4791 00000100:00080000:3.0:1769077803.679970:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077803.679975:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077803.679976:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077803.679983:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077803.679993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612744192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077803.680016:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff8b10037ded80 x1855007612743808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077803.680031:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612743808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077803.680039:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077803.680042:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077803.680045:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612743808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077803.680063:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743936:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077803.680078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077803.680087:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff8b10037de680 x1855007612743936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077803.680095:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612743936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077803.680101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077803.680103:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077803.680105:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612743936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077803.680109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077803.680129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077803.680136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff8b10037dc380 x1855007612744064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077803.680144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612744064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077803.680150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077803.680152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077803.680154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077803.680158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077803.680169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077803.680178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff8b10037dd180 x1855007612744192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077803.680192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612744192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077803.680208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077803.680212:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077803.680216:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077804.575733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077804.575890:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077805.599709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077805.599810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077806.623777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077806.623901:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077807.647793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077807.647867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077808.671851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077808.671868:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077808.799762:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077808.799773:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077808.799778:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077808.799805:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4796 00000100:00080000:2.0:1769077808.799821:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077808.799829:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077808.799876:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612744320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077808.799915:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077808.799918:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077808.799920:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077808.799923:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4796 00000100:00080000:2.0:1769077808.799925:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077808.799928:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077808.799938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077808.799948:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612744448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077808.799957:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077808.799961:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077808.799962:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077808.799965:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4796 00000100:00080000:2.0:1769077808.799968:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077808.799971:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077808.799974:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077808.799982:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612744576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077808.799995:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612744320:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077808.799996:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077808.800014:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077808.800016:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077808.800020:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4796 00000100:00080000:2.0:1769077808.800023:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077808.800027:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077808.800031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077808.800041:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612744704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077808.800079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077808.800112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff8b1005d58e00 x1855007612744320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077808.800126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612744320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077808.800135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077808.800139:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077808.800142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612744320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077808.800148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077808.800163:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077808.800171:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff8b1005d5a300 x1855007612744448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077808.800179:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612744448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077808.800188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077808.800191:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077808.800193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077808.800215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077808.800226:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077808.800233:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff8b1005d58000 x1855007612744576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077808.800240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612744576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077808.800246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077808.800249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077808.800251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077808.800255:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077808.800266:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077808.800272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff8b1005d58a80 x1855007612744704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077808.800279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612744704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077808.800284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077808.800286:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077808.800289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077809.695821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077809.695888:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077810.719779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077810.719853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077811.743778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077811.743855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077812.767767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077812.767844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077813.791760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077813.791860:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077813.919806:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077813.919822:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077813.919829:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077813.919841:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4802 00000100:00080000:3.0:1769077813.919862:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077813.919870:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077813.919916:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612744832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077813.919944:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077813.919949:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077813.919951:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077813.919956:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4802 00000100:00080000:3.0:1769077813.919960:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077813.919965:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077813.919978:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077813.919996:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612744960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077813.920012:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077813.920014:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612744832:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077813.920016:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077813.920019:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077813.920032:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4802 00000100:00080000:3.0:1769077813.920036:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077813.920039:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077813.920046:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077813.920060:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612745088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077813.920076:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077813.920079:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077813.920081:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077813.920085:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4802 00000100:00080000:3.0:1769077813.920089:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077813.920091:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769077813.920101:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769077813.920104:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077813.920118:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612745216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077813.920162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff8b10037dca80 x1855007612744832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077813.920179:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612744832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077813.920188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077813.920192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077813.920195:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612744832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077813.920215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769077813.920233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077813.920242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff8b10037dd180 x1855007612744960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077813.920250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612744960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077813.920256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077813.920258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077813.920261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612744960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769077813.920265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077813.920276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077813.920283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff8b10037dc380 x1855007612745088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077813.920291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612745088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077813.920296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077813.920298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077813.920300:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077813.920303:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769077813.920314:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077813.920320:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff8b10037de680 x1855007612745216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077813.920328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612745216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077813.920333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077813.920335:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769077813.920337:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077814.815741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077814.815921:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077815.839822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077815.839827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077816.863739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077816.863862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077817.887794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077817.887937:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077818.911720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077818.911783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077819.039778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077819.039787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077819.039790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077819.039798:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4807 00000100:00080000:1.0:1769077819.039809:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077819.039815:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077819.039848:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612745344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077819.039869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077819.039872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077819.039873:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077819.039877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4807 00000100:00080000:1.0:1769077819.039880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077819.039883:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077819.039891:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077819.039901:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612745344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077819.039902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612745472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077819.039918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077819.039920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077819.039921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077819.039924:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4807 00000100:00080000:1.0:1769077819.039926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077819.039929:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077819.039932:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077819.039940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612745600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077819.039947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077819.039949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077819.039950:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077819.039953:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4807 00000100:00080000:1.0:1769077819.039956:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077819.039958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077819.039961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077819.039968:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612745728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077819.039979:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077819.040017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff8b1006136d80 x1855007612745344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077819.040036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612745344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077819.040047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077819.040051:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077819.040056:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612745344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077819.040079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745472:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077819.040105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077819.040120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff8b1006136680 x1855007612745472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077819.040133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612745472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077819.040142:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077819.040145:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077819.040150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077819.040156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077819.040179:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077819.040191:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff8b1006137100 x1855007612745600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077819.040204:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612745600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077819.040213:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077819.040216:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077819.040220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077819.040227:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077819.040248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077819.040259:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077819/real 1769077819] req@ffff8b1006134e00 x1855007612745728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077819.040272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612745728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077819.040281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077819.040285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077819.040288:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077819.935744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077819.935799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077820.959704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077820.959778:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077821.983766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077821.983823:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077823.007764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077823.007778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077824.031758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077824.031820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077824.159700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077824.159709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077824.159712:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077824.159721:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4812 00000100:00080000:1.0:1769077824.159730:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077824.159736:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077824.159780:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612745856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077824.159804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077824.159806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077824.159807:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077824.159810:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4812 00000100:00080000:1.0:1769077824.159813:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077824.159815:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077824.159823:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077824.159835:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612745984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077824.159843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077824.159847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077824.159849:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077824.159852:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612745856:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077824.159855:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4812 00000100:00080000:1.0:1769077824.159861:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077824.159865:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077824.159869:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077824.159880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612746112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077824.159888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077824.159891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077824.159893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077824.159895:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4812 00000100:00080000:1.0:1769077824.159898:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077824.159901:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077824.159904:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077824.159911:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612746240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077824.159932:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077824.159971:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff8b1006134a80 x1855007612745856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077824.159996:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612745856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077824.160008:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077824.160013:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077824.160016:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612745856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077824.160050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077824.160078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077824.160093:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff8b1006137480 x1855007612745984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077824.160107:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612745984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077824.160113:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077824.160116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077824.160120:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612745984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077824.160128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077824.160156:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077824.160170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff8b1006134700 x1855007612746112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077824.160185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612746112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077824.160196:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077824.160200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077824.160204:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077824.160213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077824.160244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077824.160260:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff8b1006134e00 x1855007612746240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077824.160271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612746240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077824.160281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077824.160285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077824.160289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077825.055746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077825.055816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077826.079762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077826.079830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077827.103744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077827.103792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077828.127717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077828.127778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077829.151752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077829.151814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077829.279758:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.279769:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077829.279774:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.279785:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4817 00000100:00080000:2.0:1769077829.279797:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.279805:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077829.279845:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612746368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.279888:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.279898:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077829.279900:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.279906:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4817 00000100:00080000:2.0:1769077829.279911:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.279916:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077829.279927:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077829.279930:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612746368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077829.279945:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612746496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.279959:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.279963:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077829.279965:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.279969:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4817 00000100:00080000:2.0:1769077829.279975:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.279980:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077829.279987:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077829.280001:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612746624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.280012:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.280016:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769077829.280019:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746496:192.168.204.102@tcp:38:kworker.0 00000100:00080000:2.0:1769077829.280020:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.280029:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4817 00000100:00080000:2.0:1769077829.280034:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.280039:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077829.280046:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077829.280046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077829.280067:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612746752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077829.280109:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077830.175809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077830.175812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077831.199744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077831.199795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077832.223762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077832.223821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077832.351919:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:0.0:1769077832.351957:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077832.351973:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077832.352008:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077832.352014:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769077832.352022:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769077832.352084:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077832] req@ffff8b1005d58000 x1855007612746752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077832.352111:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612746752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077832.352122:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077832.352129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077832.352133:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077832.352142:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077832] req@ffff8b1005d59180 x1855007612746496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077832.352154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612746496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077832.352162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077832.352166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077832.352169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077832.352176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077832] req@ffff8b1005d58a80 x1855007612746624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077832.352188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612746624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077832.352197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077832.352201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077832.352204:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612746624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077832.352218:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077832] req@ffff8b1005d5b800 x1855007612746368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077832.352233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612746368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077832.352241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077832.352245:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077832.352249:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612746368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077833.247762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077833.247818:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077834.271759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077834.271818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077834.399786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077834.399795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077834.399798:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077834.399806:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4822 00000100:00080000:1.0:1769077834.399816:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077834.399821:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077834.399850:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612746880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077834.399876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077834.399879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077834.399880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077834.399884:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4822 00000100:00080000:1.0:1769077834.399888:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077834.399892:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077834.399902:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077834.399916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612747008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077834.399928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077834.399932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077834.399935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077834.399938:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612746880:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077834.399942:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4822 00000100:00080000:1.0:1769077834.399949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077834.399954:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077834.399960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077834.399974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612747136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077834.399985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077834.399988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077834.399991:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077834.399994:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4822 00000100:00080000:1.0:1769077834.399999:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077834.400004:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077834.400009:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769077834.400018:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769077834.400024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612747264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077834.400058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff8b1006137100 x1855007612746880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077834.400079:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612746880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077834.400091:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077834.400096:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077834.400101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612746880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077834.400128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077834.400161:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077834.400177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff8b1006136680 x1855007612747008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077834.400191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612747008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077834.400201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077834.400204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077834.400208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747008:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077834.400215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077834.400239:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077834.400253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff8b1006136d80 x1855007612747136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077834.400277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612747136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077834.400287:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077834.400290:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077834.400293:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077834.400300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077834.400321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077834.400333:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff8b1006135500 x1855007612747264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077834.400345:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612747264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077834.400353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077834.400356:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077834.400374:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077835.295748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077835.295811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077836.319750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077836.319820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077837.343841:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077837.343841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077838.367729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077838.367783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077839.392081:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077839.392145:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077839.519742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077839.519757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077839.519761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077839.519769:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4827 00000100:00080000:2.0:1769077839.519779:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077839.519784:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077839.519815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612747392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077839.519840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077839.519843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077839.519844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077839.519847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4827 00000100:00080000:2.0:1769077839.519849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077839.519852:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077839.519859:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077839.519869:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612747520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077839.519878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077839.519880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077839.519882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077839.519884:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4827 00000100:00080000:2.0:1769077839.519886:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077839.519889:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077839.519892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077839.519899:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612747392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077839.519902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612747648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077839.519916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077839.519919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077839.519920:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077839.519923:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4827 00000100:00080000:2.0:1769077839.519925:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077839.519928:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077839.519931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077839.519939:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612747776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077839.519961:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077839.519991:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff8b1005d5a300 x1855007612747392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077839.520005:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612747392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077839.520013:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077839.520016:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077839.520019:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612747392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077839.520026:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077839.520041:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077839.520049:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff8b1005d58e00 x1855007612747520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077839.520058:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612747520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077839.520064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077839.520066:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077839.520068:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077839.520087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077839.520110:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077839.520117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff8b1005d5b480 x1855007612747648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077839.520125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612747648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077839.520131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077839.520133:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077839.520135:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077839.520139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077839.520149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077839.520155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff8b1005d5bb80 x1855007612747776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077839.520163:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612747776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077839.520169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077839.520170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077839.520172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612747776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077840.415725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077840.415804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077841.439784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077841.439823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077842.463696:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077842.463776:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077843.487749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077843.487809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077844.511784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077844.511826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077844.639747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077844.639756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077844.639759:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077844.639768:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4832 00000100:00080000:2.0:1769077844.639777:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077844.639783:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077844.639815:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612747904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077844.639840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077844.639842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077844.639844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077844.639847:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4832 00000100:00080000:2.0:1769077844.639849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077844.639851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077844.639858:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077844.639866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612748032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077844.639873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077844.639876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077844.639878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077844.639880:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4832 00000100:00080000:2.0:1769077844.639882:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077844.639885:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077844.639889:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077844.639893:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612747904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077844.639900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612748160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077844.639908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077844.639910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077844.639911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077844.639914:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4832 00000100:00080000:2.0:1769077844.639916:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077844.639918:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077844.639921:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077844.639930:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612748288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077844.639955:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077844.639985:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077844] req@ffff8b1005d59880 x1855007612747904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077844.639999:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612747904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077844.640006:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077844.640009:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077844.640012:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612747904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077844.640017:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748032:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077844.640031:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077844.640039:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077844] req@ffff8b1005d59500 x1855007612748032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077844.640052:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612748032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077844.640057:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077844.640059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077844.640061:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077844.640080:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077844.640092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077844.640098:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077844] req@ffff8b1005d5aa00 x1855007612748160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077844.640106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612748160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077844.640111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077844.640113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077844.640115:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077844.640119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077844.640129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077844.640135:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077844] req@ffff8b1005d5b100 x1855007612748288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077844.640143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612748288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077844.640152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077844.640154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077844.640157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077845.535741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077845.535828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077846.559712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077846.559805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077847.583749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077847.583811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077848.607738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077848.607851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077849.631739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077849.631827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077849.759786:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077849.759797:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077849.759801:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077849.759809:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4837 00000100:00080000:3.0:1769077849.759818:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077849.759823:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077849.759855:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612748416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077849.759879:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077849.759888:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077849.759890:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077849.759893:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4837 00000100:00080000:3.0:1769077849.759896:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077849.759900:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077849.759907:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077849.759928:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612748416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077849.759931:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612748544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077849.759941:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077849.759944:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077849.759946:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077849.759948:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4837 00000100:00080000:3.0:1769077849.759951:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077849.759953:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077849.759957:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077849.759973:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612748672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077849.759980:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077849.759983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077849.759994:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077849.759997:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4837 00000800:00000100:2.0:1769077849.760001:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077849.760001:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077849.760008:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077849.760012:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077849.760021:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612748800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077849.760040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff8b10037ded80 x1855007612748416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077849.760065:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612748416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077849.760074:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077849.760077:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077849.760080:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612748416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077849.760121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077849.760150:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077849.760174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff8b10037de300 x1855007612748544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077849.760187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612748544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077849.760193:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077849.760195:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077849.760197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077849.760202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077849.760218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077849.760226:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff8b10037ddf80 x1855007612748672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077849.760234:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612748672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077849.760239:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077849.760241:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077849.760243:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077849.760246:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077849.760257:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077849.760264:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff8b10037dd500 x1855007612748800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077849.760271:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612748800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077849.760276:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077849.760278:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077849.760280:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612748800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077850.655722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077850.655817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077851.679808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077851.679813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077852.703792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077852.703943:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077853.727732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077853.727794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077854.751773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077854.751877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077854.879816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077854.879826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077854.879830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077854.879838:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4842 00000100:00080000:2.0:1769077854.879848:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077854.879853:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077854.879895:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612748928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077854.879918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077854.879921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077854.879922:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077854.879925:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4842 00000100:00080000:2.0:1769077854.879928:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077854.879931:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077854.879939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077854.879948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077854.879956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077854.879958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077854.879959:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077854.879961:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4842 00000100:00080000:2.0:1769077854.879963:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077854.879965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077854.879968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077854.879974:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612748928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077854.879979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612749184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077854.879992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077854.879994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077854.879995:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077854.879998:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4842 00000100:00080000:2.0:1769077854.880000:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077854.880002:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077854.880005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077854.880012:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612749312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077854.880065:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077854.880102:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff8b1005d5ad80 x1855007612748928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.880121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612748928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.880133:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.880138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077854.880142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612748928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077854.880167:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749056:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077854.880196:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077854.880206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff8b1005d58380 x1855007612749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.880216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612749056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.880221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.880224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077854.880226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077854.880230:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077854.880243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077854.880249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff8b1005d5b100 x1855007612749184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.880262:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612749184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.880268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.880269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077854.880271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077854.880275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077854.880286:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077854.880292:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff8b1005d5aa00 x1855007612749312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.880300:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612749312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.880305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.880307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077854.880309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077855.775739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077855.775831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077856.799810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077856.799894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077857.823718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077857.823811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077858.847768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077858.847793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077859.871704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077859.871793:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077859.999771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.999780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077859.999784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.999792:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4848 00000100:00080000:3.0:1769077859.999802:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.999810:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077859.999846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612749440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.999879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.999883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077859.999885:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.999891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4848 00000100:00080000:3.0:1769077859.999895:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.999900:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077859.999909:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612749440:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769077859.999910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077859.999945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612749568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.999958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.999962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077859.999964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.999970:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4848 00000100:00080000:3.0:1769077859.999975:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.999979:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077859.999986:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077860.000001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612749696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077860.000014:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077860.000017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077860.000021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077860.000023:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077860.000027:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4848 00000100:00080000:3.0:1769077860.000031:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077860.000035:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077860.000039:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077860.000049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612749824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077860.000055:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077860] req@ffff8b10037dc000 x1855007612749440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.000078:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612749440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.000090:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.000095:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077860.000100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612749440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077860.000124:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077860.000147:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077860.000157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077860] req@ffff8b10037dd880 x1855007612749568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.000168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612749568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.000174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.000177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077860.000180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077860.000186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077860.000214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077860.000221:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077860] req@ffff8b10037df100 x1855007612749696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.000229:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612749696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.000234:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.000236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077860.000239:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077860.000242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077860.000253:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077860.000259:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077860] req@ffff8b10037ddc00 x1855007612749824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077860.000267:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612749824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077860.000272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077860.000274:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077860.000275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612749824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077860.895764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077860.895824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077861.919722:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077861.919795:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077862.943751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077862.943809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077863.967751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077863.967833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077864.991788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077864.991900:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077865.119712:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.119722:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077865.119725:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.119733:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4853 00000100:00080000:3.0:1769077865.119743:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.119748:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077865.119780:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612749952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.119803:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.119806:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077865.119807:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.119810:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4853 00000100:00080000:3.0:1769077865.119812:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.119816:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077865.119824:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077865.119835:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612749952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077865.119836:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612750080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.119865:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.119868:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077865.119870:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.119872:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4853 00000100:00080000:3.0:1769077865.119875:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.119878:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077865.119883:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077865.119901:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612750208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.119912:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.119916:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077865.119918:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077865.119923:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077865.119927:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4853 00000100:00080000:3.0:1769077865.119934:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.119939:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077865.119944:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077865.119953:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff8b10037df800 x1855007612749952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077865.119957:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612750336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077865.119985:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612749952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.119995:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077865.119999:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077865.120001:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612749952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077865.120018:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077865.120033:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077865.120042:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff8b10037dc700 x1855007612750080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077865.120050:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612750080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.120056:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077865.120058:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077865.120060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077865.120064:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077865.120077:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077865.120083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff8b10037dce00 x1855007612750208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077865.120092:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612750208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.120101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077865.120104:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077865.120108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077865.120114:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077865.120134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077865.120146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff8b10037df480 x1855007612750336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077865.120159:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612750336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077865.120165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077865.120167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077865.120169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077866.015808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077866.015837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077867.039739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077867.039831:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077868.063783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077868.063861:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077869.087733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077869.087839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077870.111755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077870.111813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077870.239746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077870.239754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077870.239771:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077870.239780:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4858 00000100:00080000:3.0:1769077870.239790:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077870.239796:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077870.239830:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612750464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077870.239854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077870.239857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077870.239858:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077870.239861:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4858 00000100:00080000:3.0:1769077870.239863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077870.239866:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077870.239873:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077870.239883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612750592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077870.239890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077870.239893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077870.239896:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077870.239900:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4858 00000100:00080000:3.0:1769077870.239904:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077870.239908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077870.239916:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077870.239920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612750464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077870.239931:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612750720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077870.239945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077870.239949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077870.239951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077870.239954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4858 00000100:00080000:3.0:1769077870.239958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077870.239963:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077870.239968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077870.239981:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612750848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077870.239993:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077870.240031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff8b10037dea00 x1855007612750464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077870.240051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612750464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077870.240072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077870.240077:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077870.240081:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612750464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077870.240089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750592:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077870.240113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077870.240126:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff8b10037df480 x1855007612750592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077870.240140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612750592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077870.240148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077870.240152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077870.240156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077870.240178:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077870.240200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077870.240213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff8b10037dce00 x1855007612750720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077870.240232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612750720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077870.240241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077870.240244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077870.240248:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077870.240254:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077870.240272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077870.240283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff8b10037dc700 x1855007612750848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077870.240295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612750848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077870.240303:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077870.240306:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077870.240310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612750848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077871.135724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077871.135815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077872.159780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077872.159885:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077873.183806:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077873.183807:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077874.207763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077874.207829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077875.231740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077875.231809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077875.359775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.359788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077875.359792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.359803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4863 00000100:00080000:3.0:1769077875.359815:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.359820:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077875.359861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612750976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077875.359885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.359888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077875.359890:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.359893:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4863 00000100:00080000:3.0:1769077875.359895:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.359898:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077875.359905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077875.359913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077875.359921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.359923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077875.359925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.359927:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4863 00000100:00080000:3.0:1769077875.359929:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.359932:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077875.359935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077875.359942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612751232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077875.359947:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612750976:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077875.359951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.359959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077875.359961:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.359964:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4863 00000100:00080000:3.0:1769077875.359965:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.359974:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077875.359978:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077875.359989:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612751360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077875.360023:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077875.360070:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff8b10037df800 x1855007612750976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077875.360091:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612750976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077875.360102:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077875.360107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077875.360111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612750976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077875.360120:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077875.360146:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077875.360159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff8b10037ddc00 x1855007612751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077875.360174:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612751104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077875.360183:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077875.360186:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077875.360190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077875.360215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077875.360238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077875.360250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff8b10037df100 x1855007612751232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077875.360263:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612751232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077875.360278:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077875.360282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077875.360286:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077875.360293:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077875.360313:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077875.360326:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff8b10037dd880 x1855007612751360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077875.360340:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612751360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077875.360348:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077875.360352:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077875.360356:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077876.255773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077876.255849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077877.279726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077877.279791:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077878.303747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077878.303810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077879.327716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077879.327784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077880.352780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077880.353604:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077880.479713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.479722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077880.479726:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.479735:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4868 00000100:00080000:3.0:1769077880.479745:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.479751:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077880.479784:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612751488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.479810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.479813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077880.479815:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.479819:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4868 00000100:00080000:3.0:1769077880.479822:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.479826:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077880.479844:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077880.479849:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612751488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077880.479861:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612751616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.479872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.479876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077880.479878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.479888:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4868 00000100:00080000:3.0:1769077880.479895:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.479899:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077880.479905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077880.479918:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077880.479918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612751744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.479931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.479936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077880.479938:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.479942:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4868 00000100:00080000:3.0:1769077880.479946:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.479950:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077880.479956:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff8b10037dc000 x1855007612751488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077880.479956:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077880.479970:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612751872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077880.479979:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612751488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077880.479991:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077880.479996:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077880.480000:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612751488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077880.480021:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077880.480039:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077880.480050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff8b10037dd500 x1855007612751616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077880.480062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612751616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077880.480069:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077880.480072:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077880.480075:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077880.480081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077880.480096:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077880.480104:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff8b10037ddf80 x1855007612751744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077880.480115:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612751744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077880.480122:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077880.480125:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077880.480128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077880.480134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077880.480148:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077880.480157:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff8b10037de300 x1855007612751872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077880.480167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612751872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077880.480175:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077880.480177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077880.480181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612751872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077881.375775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077881.375999:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077882.399750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077882.399817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077883.423779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077883.423855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077884.447760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077884.447827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077885.471750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077885.471827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077885.599746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.599756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077885.599760:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.599769:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4873 00000100:00080000:3.0:1769077885.599780:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.599786:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077885.599820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612752000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.599850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.599854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077885.599856:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.599860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4873 00000100:00080000:3.0:1769077885.599863:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.599867:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077885.599876:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077885.599900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612752128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077885.599904:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612752000:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077885.599918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.599922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077885.599924:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.599929:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4873 00000100:00080000:3.0:1769077885.599933:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.599936:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077885.599941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077885.599952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612752256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.599961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.599965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077885.599967:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077885.599969:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077885.599973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4873 00000100:00080000:3.0:1769077885.599979:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.599983:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077885.599988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077885.600000:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612752384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077885.600007:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff8b10037ded80 x1855007612752000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077885.600026:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612752000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077885.600037:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077885.600041:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077885.600046:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612752000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077885.600066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752128:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077885.600084:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077885.600094:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff8b10037de680 x1855007612752128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077885.600107:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612752128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077885.600114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077885.600117:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077885.600121:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077885.600126:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077885.600141:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077885.600150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff8b10037dc380 x1855007612752256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077885.600161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612752256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077885.600168:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077885.600171:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077885.600174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077885.600180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077885.600194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077885.600202:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff8b10037dd180 x1855007612752384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077885.600213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612752384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077885.600221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077885.600224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077885.600226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077886.495756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077886.495808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077887.519802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077887.519986:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077888.543738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077888.543746:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077889.567726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077889.567753:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077890.591732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077890.591792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077890.719757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077890.719767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077890.719771:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077890.719779:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4878 00000100:00080000:3.0:1769077890.719788:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077890.719793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077890.719824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612752512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077890.719844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077890.719847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077890.719848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077890.719852:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4878 00000100:00080000:3.0:1769077890.719854:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077890.719857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077890.719865:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077890.719875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612752640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077890.719884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077890.719886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077890.719887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077890.719892:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612752512:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077890.719892:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4878 00000100:00080000:3.0:1769077890.719896:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077890.719900:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077890.719903:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077890.719910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612752768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077890.719916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077890.719918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077890.719919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077890.719921:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4878 00000100:00080000:3.0:1769077890.719923:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077890.719925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077890.719928:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077890.719934:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612752896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077890.719958:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077890.719987:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff8b10037dca80 x1855007612752512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077890.720002:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612752512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077890.720011:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077890.720015:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077890.720019:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612752512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077890.720050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077890.720068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077890.720076:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff8b10037dd180 x1855007612752640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077890.720084:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612752640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077890.720090:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077890.720092:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077890.720094:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077890.720098:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077890.720109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077890.720114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff8b10037dc380 x1855007612752768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077890.720127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612752768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077890.720132:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077890.720134:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077890.720136:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077890.720139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077890.720149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077890.720155:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff8b10037de680 x1855007612752896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077890.720161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612752896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077890.720170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077890.720172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077890.720174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612752896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077891.615749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077891.615802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077892.639718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077892.639785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077893.663743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077893.663805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769077894.687745:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077894.687770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077895.711714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077895.711767:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077895.839756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077895.839765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077895.839768:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077895.839776:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4883 00000100:00080000:2.0:1769077895.839785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077895.839791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077895.839819:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612753024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077895.839840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077895.839842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077895.839843:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077895.839845:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4883 00000100:00080000:2.0:1769077895.839847:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077895.839850:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077895.839858:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077895.839867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612753152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077895.839875:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612753024:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077895.839882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077895.839885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077895.839886:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077895.839890:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4883 00000100:00080000:2.0:1769077895.839894:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077895.839897:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077895.839901:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077895.839909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612753280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077895.839918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077895.839921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077895.839923:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077895.839927:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4883 00000100:00080000:2.0:1769077895.839932:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077895.839935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077895.839941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077895.839952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612753408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077895.840057:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077895.840085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077895.840102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077896.735714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077896.735757:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077897.759786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077897.759836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077898.783700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077898.783765:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769077898.911940:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769077898.911980:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769077898.911999:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077898.912046:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077898.912056:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769077898.912064:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769077898.912250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077898] req@ffff8b1005d59880 x1855007612753152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077898.912308:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612753152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077898.912320:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077898.912326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077898.912330:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077898.912338:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077898] req@ffff8b1005d5bb80 x1855007612753280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077898.912352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612753280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077898.912360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077898.912364:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077898.912367:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077898.912374:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077898] req@ffff8b1005d5b480 x1855007612753408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077898.912385:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612753408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077898.912393:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077898.912397:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077898.912400:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077898.912407:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077898] req@ffff8b1005d59500 x1855007612753024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077898.912424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612753024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077898.912432:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077898.912436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077898.912439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612753024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077899.807689:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077899.807758:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077900.831711:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077900.831816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077900.959772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.959785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077900.959791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.959804:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4889 00000100:00080000:3.0:1769077900.959818:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.959827:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077900.959872:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612753536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.959920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.959924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077900.959926:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.959931:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4889 00000100:00080000:3.0:1769077900.959936:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.959941:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077900.959955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077900.959970:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612753536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077900.959973:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.959997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.960001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077900.960003:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.960010:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4889 00000100:00080000:3.0:1769077900.960014:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.960018:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077900.960024:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077900.960040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612753792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.960051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.960055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077900.960059:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077900.960066:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077900.960071:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4889 00000100:00080000:3.0:1769077900.960078:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.960085:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077900.960094:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077900.960110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612753920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077900.960124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff8b10037ded80 x1855007612753536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077900.960156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612753536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077900.960172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077900.960178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077900.960183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612753536:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077900.960215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753664:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077900.960277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077900.960292:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff8b10037de300 x1855007612753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077900.960306:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612753664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077900.960315:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077900.960318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077900.960321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077900.960340:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077900.960366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077900.960380:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff8b10037ddf80 x1855007612753792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077900.960391:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612753792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077900.960417:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077900.960421:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077900.960424:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077900.960431:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077900.960453:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077900.960465:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff8b10037dd500 x1855007612753920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077900.960477:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612753920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077900.960485:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077900.960496:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077900.960500:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612753920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077901.855780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077901.855844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077902.879821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077902.879829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077903.903784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077903.903849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077904.927730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077904.927783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077905.951742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077905.951808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077906.079788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.079812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077906.079818:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.079829:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4894 00000100:00080000:3.0:1769077906.079843:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.079850:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077906.079908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612754048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077906.079938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.079941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077906.079943:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.079946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4894 00000100:00080000:3.0:1769077906.079949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.079952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077906.079962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077906.079973:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612754176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077906.079980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.079983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077906.079984:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.079987:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4894 00000100:00080000:3.0:1769077906.079989:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.079991:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077906.079994:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077906.080001:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612754304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077906.080007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.080010:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077906.080012:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.080016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4894 00000100:00080000:3.0:1769077906.080019:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.080022:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077906.080024:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077906.080039:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612754432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077906.080115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612754048:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769077906.080236:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077906.080279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff8b10037dc000 x1855007612754048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077906.080314:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612754048/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077906.080324:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077906.080329:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077906.080332:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612754048:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077906.080339:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754176:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077906.080359:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077906.080368:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff8b10037dd880 x1855007612754176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077906.080378:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612754176/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077906.080384:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077906.080387:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077906.080390:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754176:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077906.080394:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077906.080407:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077906.080414:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff8b10037df100 x1855007612754304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077906.080423:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612754304/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077906.080429:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077906.080431:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077906.080433:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077906.080438:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077906.080451:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077906.080458:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff8b10037ddc00 x1855007612754432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077906.080467:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612754432/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077906.080473:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077906.080475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077906.080477:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077906.975743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077906.975832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077907.999739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077907.999829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077909.023764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077909.023830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077910.047786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077910.047849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077911.071801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077911.071872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077911.199745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.199757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077911.199762:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.199773:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4899 00000100:00080000:3.0:1769077911.199786:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.199793:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077911.199840:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612754560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.199869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.199873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077911.199874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.199877:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4899 00000100:00080000:3.0:1769077911.199880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.199883:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077911.199892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077911.199902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612754688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.199910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.199913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077911.199914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.199917:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4899 00000100:00080000:3.0:1769077911.199919:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.199921:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077911.199924:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077911.199928:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612754560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077911.199935:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612754816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.199944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.199946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077911.199947:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.199950:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4899 00000100:00080000:3.0:1769077911.199953:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.199955:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077911.199958:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077911.199966:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612754944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077911.200020:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077911.200058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077911] req@ffff8b10037df800 x1855007612754560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077911.200092:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612754560/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077911.200101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077911.200105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077911.200108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612754560:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077911.200114:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754688:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077911.200134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077911.200146:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077911] req@ffff8b10037dc700 x1855007612754688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077911.200168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612754688/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077911.200178:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077911.200181:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077911.200186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754688:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077911.200212:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077911.200231:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077911.200240:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077911] req@ffff8b10037dce00 x1855007612754816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077911.200249:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612754816/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077911.200254:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077911.200257:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077911.200259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077911.200263:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077911.200274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077911.200281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077911] req@ffff8b10037df480 x1855007612754944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077911.200289:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612754944/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077911.200299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077911.200301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077911.200304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612754944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077912.095735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077912.095806:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077913.119755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077913.119826:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077914.143714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077914.143769:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077915.167779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077915.167867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077916.191732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077916.191796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077916.319764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077916.319781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077916.319786:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077916.319796:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4904 00000100:00080000:3.0:1769077916.319807:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077916.319814:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077916.319852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612755072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077916.319883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077916.319887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077916.319889:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077916.319894:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4904 00000100:00080000:3.0:1769077916.319897:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077916.319902:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077916.319912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077916.319927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612755200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077916.319939:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612755072:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077916.319941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077916.319949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077916.319951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077916.319956:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4904 00000100:00080000:3.0:1769077916.319960:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077916.319975:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077916.319996:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077916.320010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612755328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077916.320020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077916.320024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077916.320027:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077916.320030:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077916.320035:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4904 00000100:00080000:3.0:1769077916.320041:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077916.320045:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077916.320051:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077916.320064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612755456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077916.320066:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff8b10037dea00 x1855007612755072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077916.320099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612755072/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077916.320109:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077916.320112:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077916.320116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612755072:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077916.320136:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755200:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077916.320151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077916.320159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff8b10037df480 x1855007612755200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077916.320167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612755200/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077916.320172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077916.320175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077916.320177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755200:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077916.320181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077916.320193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077916.320198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff8b10037dce00 x1855007612755328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077916.320206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612755328/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077916.320211:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077916.320213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077916.320215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077916.320218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077916.320229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077916.320235:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff8b10037dc700 x1855007612755456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077916.320242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612755456/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077916.320247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077916.320249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077916.320251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077917.215752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077917.215803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077918.239714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077918.239765:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077919.263846:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077919.263883:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077920.287708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077920.287816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077921.311779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077921.311843:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077921.439712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.439724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077921.439728:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.439738:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4909 00000100:00080000:3.0:1769077921.439748:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.439756:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077921.439792:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612755584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077921.439823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.439827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077921.439829:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.439833:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4909 00000100:00080000:3.0:1769077921.439837:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.439841:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077921.439850:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077921.439863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612755712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077921.439875:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612755584:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077921.439876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.439885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077921.439887:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.439893:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4909 00000100:00080000:3.0:1769077921.439924:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.439928:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077921.439934:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077921.439946:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612755840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077921.439956:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077921.439972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.439976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077921.439979:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.439985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4909 00000100:00080000:3.0:1769077921.439990:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077921.439991:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff8b10037df800 x1855007612755584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077921.439997:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077921.440007:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077921.440022:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612755584/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077921.440023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612755968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077921.440055:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077921.440062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077921.440066:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612755584:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077921.440087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755712:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077921.440106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077921.440116:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff8b10037ddc00 x1855007612755712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077921.440131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612755712/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077921.440140:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077921.440143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077921.440147:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755712:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077921.440154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077921.440176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077921.440186:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff8b10037df100 x1855007612755840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077921.440195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612755840/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077921.440200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077921.440203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077921.440205:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077921.440209:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077921.440222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077921.440228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff8b10037dd880 x1855007612755968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077921.440236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612755968/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077921.440241:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077921.440243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077921.440245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612755968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077922.335784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077922.335867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077923.359756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077923.359820:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077924.383782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077924.383839:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077925.407789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077925.407862:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077926.431746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077926.431833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077926.559799:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077926.559839:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077926.559844:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077926.559855:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4914 00000100:00080000:1.0:1769077926.559866:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077926.559874:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077926.559913:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612756096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077926.559943:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077926.559948:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077926.559950:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077926.559955:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4914 00000100:00080000:1.0:1769077926.559959:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077926.559964:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077926.559975:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077926.559990:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612756224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077926.560001:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612756096:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077926.560007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077926.560013:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077926.560015:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077926.560020:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4914 00000100:00080000:1.0:1769077926.560025:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077926.560030:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077926.560036:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077926.560051:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612756352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077926.560062:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077926.560065:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077926.560068:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077926.560073:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4914 00000800:00000100:2.0:1769077926.560078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769077926.560081:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077926.560088:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077926.560094:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077926.560109:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612756480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077926.560123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff8b1006135880 x1855007612756096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077926.560151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612756096/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077926.560162:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077926.560173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077926.560178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612756096:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077926.560203:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756224:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769077926.560229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077926.560242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff8b1006137b80 x1855007612756224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077926.560257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612756224/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077926.560267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077926.560270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077926.560274:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756224:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769077926.560281:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077926.560304:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077926.560316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff8b1006135c00 x1855007612756352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077926.560330:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612756352/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077926.560353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077926.560357:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077926.560362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769077926.560368:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769077926.560391:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077926.560404:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff8b1006136a00 x1855007612756480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077926.560417:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612756480/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077926.560426:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077926.560429:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769077926.560433:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077927.455752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077927.455833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077928.479755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077928.479839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077929.503753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077929.503860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077930.527794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077930.527823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077931.551798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077931.551815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077931.679805:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.679813:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077931.679817:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.679825:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4919 00000100:00080000:0.0:1769077931.679835:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.679842:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077931.679872:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612756608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077931.679895:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.679898:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077931.679899:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.679902:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4919 00000100:00080000:0.0:1769077931.679905:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.679908:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077931.679915:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077931.679927:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612756736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077931.679937:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612756608:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077931.679945:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.679950:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077931.679953:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.679959:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4919 00000100:00080000:0.0:1769077931.679964:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.679969:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077931.679976:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077931.679994:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612756864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077931.680005:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769077931.680008:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.680013:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077931.680015:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.680020:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4919 00000100:00080000:0.0:1769077931.680025:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.680029:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077931.680035:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077931.680037:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff8b1006024380 x1855007612756608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077931.680063:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612756992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077931.680074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612756608/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077931.680089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077931.680095:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077931.680101:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612756608:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077931.680192:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756736:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077931.680300:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077931.680318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff8b1006024e00 x1855007612756736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077931.680370:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612756736/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077931.680378:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077931.680381:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077931.680384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756736:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077931.680389:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077931.680404:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077931.680413:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff8b1006024700 x1855007612756864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077931.680421:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612756864/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077931.680426:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077931.680428:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077931.680430:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077931.680434:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077931.680445:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077931.680452:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff8b1006027480 x1855007612756992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077931.680459:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612756992/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077931.680465:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077931.680467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077931.680468:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612756992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769077932.575745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077932.575816:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077933.599790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077933.599836:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077934.623759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077934.623816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077935.647715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077935.647792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077936.671762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077936.671826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077936.799740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077936.799748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077936.799753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077936.799762:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4924 00000100:00080000:2.0:1769077936.799771:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077936.799777:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077936.799810:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612757120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077936.799835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077936.799838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077936.799840:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077936.799842:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4924 00000100:00080000:2.0:1769077936.799845:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077936.799847:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077936.799855:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077936.799866:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612757248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077936.799874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077936.799876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077936.799878:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077936.799882:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612757120:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077936.799883:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4924 00000100:00080000:2.0:1769077936.799889:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077936.799893:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077936.799896:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077936.799906:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612757376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077936.799912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077936.799915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077936.799917:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077936.799920:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4924 00000100:00080000:2.0:1769077936.799932:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077936.799935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077936.799938:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769077936.799945:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769077936.799948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612757504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077936.799982:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff8b1005d58e00 x1855007612757120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077936.799997:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612757120/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077936.800004:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077936.800007:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077936.800010:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612757120:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077936.800029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757248:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077936.800045:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077936.800053:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff8b1005d5a300 x1855007612757248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077936.800062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612757248/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077936.800067:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077936.800070:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077936.800072:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757248:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077936.800076:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757376:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077936.800087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077936.800093:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff8b1005d5b800 x1855007612757376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077936.800101:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612757376/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077936.800106:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077936.800107:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077936.800109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077936.800113:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077936.800122:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077936.800128:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff8b1005d58a80 x1855007612757504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077936.800136:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612757504/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077936.800141:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077936.800143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077936.800145:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077937.695738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077937.695790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077938.719756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077938.719865:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077939.743698:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077939.743769:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077940.767726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077940.767784:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077941.791710:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077941.791792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077941.919739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077941.919747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077941.919751:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077941.919759:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4930 00000100:00080000:2.0:1769077941.919769:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077941.919774:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077941.919807:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612757632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077941.919837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077941.919841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077941.919843:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077941.919848:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4930 00000100:00080000:2.0:1769077941.919852:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077941.919857:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077941.919866:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077941.919878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612757760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077941.919887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077941.919890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077941.919891:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077941.919897:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4930 00000100:00100000:3.0:1769077941.919899:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612757632:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077941.919900:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077941.919905:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077941.919912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077941.919923:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612757888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077941.919930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077941.919933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077941.919935:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077941.919938:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4930 00000100:00080000:2.0:1769077941.919942:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077941.919946:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077941.919952:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077941.919963:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612758016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077941.919975:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077941.920017:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff8b1005d59180 x1855007612757632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077941.920036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612757632/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077941.920047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077941.920052:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077941.920056:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612757632:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077941.920081:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757760:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077941.920106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077941.920119:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff8b1005d58000 x1855007612757760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077941.920134:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612757760/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077941.920143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077941.920146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077941.920150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757760:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077941.920155:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077941.920174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077941.920185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff8b1005d58a80 x1855007612757888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077941.920198:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612757888/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077941.920206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077941.920210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077941.920214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612757888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077941.920221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077941.920241:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077941.920253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff8b1005d5b800 x1855007612758016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077941.920266:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612758016/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077941.920281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077941.920285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077941.920289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077942.815747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077942.815815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077943.839706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077943.839790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077944.863721:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077944.863793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077945.887773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077945.887864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077946.911763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077946.911855:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077947.039810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077947.039821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077947.039825:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077947.039836:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4935 00000100:00080000:0.0:1769077947.039847:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077947.039853:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077947.039886:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612758144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077947.039908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077947.039911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077947.039913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077947.039916:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4935 00000100:00080000:0.0:1769077947.039918:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077947.039921:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077947.039929:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077947.039938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612758272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077947.039946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077947.039949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077947.039951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077947.039954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4935 00000100:00080000:0.0:1769077947.039957:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077947.039960:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077947.039963:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077947.039972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612758400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077947.039979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077947.039981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077947.039983:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077947.039987:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4935 00000100:00080000:0.0:1769077947.039992:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077947.039995:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077947.039996:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612758144:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769077947.039999:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077947.040007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612758528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077947.040107:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077947.040149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff8b1006024a80 x1855007612758144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077947.040166:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612758144/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077947.040176:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077947.040180:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077947.040183:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612758144:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077947.040190:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758272:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077947.040206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077947.040214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff8b1006025f80 x1855007612758272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077947.040223:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612758272/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077947.040230:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077947.040232:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077947.040235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758272:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077947.040260:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077947.040272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077947.040279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff8b1006027b80 x1855007612758400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077947.040286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612758400/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077947.040293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077947.040295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077947.040297:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077947.040300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758528:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077947.040311:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077947.040318:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077947/real 1769077947] req@ffff8b1006026d80 x1855007612758528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077947.040326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612758528/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077947.040331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077947.040332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077947.040335:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077947.935724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077947.935770:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077948.959780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077948.959875:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077949.983757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077949.983833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077951.007753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077951.007806:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077952.031769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077952.031809:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077952.159753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077952.159763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077952.159766:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077952.159774:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4940 00000100:00080000:0.0:1769077952.159784:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077952.159790:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077952.159823:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612758656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077952.159844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077952.159847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077952.159848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077952.159851:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4940 00000100:00080000:0.0:1769077952.159853:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077952.159856:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077952.159862:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077952.159871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077952.159878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077952.159881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077952.159882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077952.159885:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4940 00000100:00080000:0.0:1769077952.159887:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077952.159891:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612758656:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077952.159892:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077952.159900:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077952.159910:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612758912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077952.159917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077952.159920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077952.159921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077952.159924:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4940 00000100:00080000:0.0:1769077952.159926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077952.159928:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077952.159931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077952.159938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612759040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077952.159961:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077952.160004:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff8b1006026300 x1855007612758656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077952.160024:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612758656/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077952.160035:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077952.160040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077952.160044:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612758656:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077952.160051:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758784:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077952.160070:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077952.160079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff8b1006026a00 x1855007612758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077952.160088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612758784/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077952.160094:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077952.160096:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077952.160099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758784:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077952.160117:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077952.160129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077952.160136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff8b1006024000 x1855007612758912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077952.160144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612758912/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077952.160149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077952.160150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077952.160152:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612758912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077952.160156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077952.160166:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077952.160172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff8b1006025880 x1855007612759040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077952.160180:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612759040/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077952.160185:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077952.160186:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077952.160188:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077953.055763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077953.055838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077954.079753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077954.079809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077955.103728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077955.103791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077956.127747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077956.127804:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077957.151725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077957.151780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077957.279741:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077957.279750:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077957.279754:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077957.279762:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4945 00000100:00080000:0.0:1769077957.279772:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077957.279777:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077957.279808:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612759168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077957.279831:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077957.279834:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077957.279835:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077957.279839:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4945 00000100:00080000:0.0:1769077957.279841:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077957.279844:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077957.279851:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077957.279859:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077957.279867:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077957.279870:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077957.279872:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077957.279875:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4945 00000100:00080000:0.0:1769077957.279878:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077957.279880:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077957.279884:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077957.279890:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612759424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077957.279899:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612759168:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077957.279900:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077957.279907:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077957.279909:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077957.279913:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4945 00000100:00080000:0.0:1769077957.279915:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077957.279918:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077957.279921:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077957.279930:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612759552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077957.279981:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077957.280020:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff8b1006025180 x1855007612759168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077957.280040:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612759168/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077957.280051:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077957.280056:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077957.280060:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612759168:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077957.280068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759296:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077957.280094:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077957.280107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff8b1006027800 x1855007612759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077957.280122:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612759296/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077957.280131:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077957.280135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077957.280139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759296:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077957.280166:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077957.280193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077957.280205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff8b1006027100 x1855007612759424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077957.280219:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612759424/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077957.280228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077957.280231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077957.280235:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077957.280242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077957.280263:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077957.280276:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff8b1006026680 x1855007612759552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077957.280289:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612759552/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077957.280314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077957.280318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077957.280322:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077958.175748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077958.175810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077959.199726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077959.199772:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077960.223773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077960.223841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077961.247768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077961.247846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077962.271755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077962.271821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077962.399727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.399737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077962.399742:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.399752:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4950 00000100:00080000:0.0:1769077962.399762:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.399803:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769077962.399841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612759680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.399864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.399867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077962.399869:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.399871:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4950 00000100:00080000:0.0:1769077962.399874:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.399877:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077962.399884:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077962.399899:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612759808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.399908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077962.399910:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612759680:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769077962.399912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077962.399916:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.399921:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4950 00000100:00080000:0.0:1769077962.399923:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.399926:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077962.399930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077962.399940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612759936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.399947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.399949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077962.399951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.399953:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4950 00000100:00080000:0.0:1769077962.399955:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.399957:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769077962.399960:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077962.399966:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612760064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077962.400006:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077962.400029:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077962.400046:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077963.295803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077963.295872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077964.319737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077964.319792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077965.343772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077965.343841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077965.472008:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769077965.472074:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077965.472107:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077965.472196:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077965.472208:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769077965.472221:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769077965.472356:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077965] req@ffff8b1006027100 x1855007612759808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077965.472395:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612759808/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077965.472426:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077965.472434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077965.472440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759808:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077965.472452:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077965] req@ffff8b1006027800 x1855007612759936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077965.472468:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612759936/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077965.472477:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077965.472481:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077965.472485:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612759936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077965.472494:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077965] req@ffff8b1006025180 x1855007612760064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077965.472516:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612760064/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077965.472525:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077965.472530:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077965.472533:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077965.472541:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077965] req@ffff8b1006026680 x1855007612759680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077965.472555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612759680/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077965.472564:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077965.472569:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077965.472573:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612759680:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077966.367757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077966.367817:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077967.391757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077967.391821:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077967.519760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077967.519769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077967.519773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077967.519782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4955 00000100:00080000:1.0:1769077967.519792:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077967.519798:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077967.519827:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612760192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077967.519853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077967.519856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077967.519857:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077967.519860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4955 00000100:00080000:1.0:1769077967.519862:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077967.519865:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077967.519872:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077967.519879:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612760320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077967.519886:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077967.519889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077967.519890:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077967.519893:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4955 00000100:00080000:1.0:1769077967.519894:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077967.519897:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077967.519903:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077967.519914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612760448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077967.519924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077967.519929:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612760192:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077967.519930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077967.519933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077967.519943:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4955 00000100:00080000:1.0:1769077967.519949:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077967.519955:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077967.519961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077967.519977:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612760576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077967.520003:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077967.520047:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff8b1006135180 x1855007612760192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077967.520069:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612760192/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077967.520083:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077967.520088:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077967.520093:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612760192:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077967.520102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760320:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077967.520149:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077967.520170:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff8b1006137800 x1855007612760320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077967.520187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612760320/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077967.520196:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077967.520201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077967.520206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760320:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077967.520233:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077967.520260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077967.520273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff8b1006136300 x1855007612760448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077967.520287:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612760448/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077967.520297:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077967.520301:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077967.520304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077967.520310:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077967.520334:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077967.520347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff8b1006136a00 x1855007612760576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077967.520358:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612760576/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077967.520372:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077967.520375:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077967.520378:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077968.415742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077968.415798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077969.439793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077969.439849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077970.463743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077970.463826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077971.487783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077971.487835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077972.511750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077972.511830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077972.639747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077972.639756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077972.639759:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077972.639769:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4960 00000100:00080000:2.0:1769077972.639781:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077972.639787:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077972.639824:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612760704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077972.639850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077972.639853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077972.639854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077972.639857:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4960 00000100:00080000:2.0:1769077972.639860:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077972.639863:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077972.639871:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077972.639880:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612760832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077972.639888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077972.639891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077972.639892:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077972.639894:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4960 00000100:00080000:2.0:1769077972.639899:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077972.639903:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077972.639909:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077972.639920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612760960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077972.639933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077972.639941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612760704:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769077972.639942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077972.639951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077972.639958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4960 00000100:00080000:2.0:1769077972.639961:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077972.639964:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077972.639970:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077972.639983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612761088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077972.640032:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077972.640105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff8b1005d5a300 x1855007612760704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077972.640122:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612760704/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077972.640132:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077972.640136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077972.640139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612760704:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077972.640146:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760832:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769077972.640172:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077972.640182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff8b1005d58e00 x1855007612760832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077972.640192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612760832/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077972.640198:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077972.640200:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077972.640203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760832:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769077972.640229:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077972.640243:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077972.640249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff8b1005d59500 x1855007612760960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077972.640257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612760960/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077972.640263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077972.640265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077972.640267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612760960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077972.640271:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769077972.640283:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769077972.640289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff8b1005d5b480 x1855007612761088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077972.640302:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612761088/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077972.640308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077972.640310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769077972.640312:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077973.535732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077973.535792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077974.559780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077974.559827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077975.583778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077975.583860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077976.607757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077976.607767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077977.631746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077977.631836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077977.759758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077977.759769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077977.759774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077977.759784:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4965 00000100:00080000:3.0:1769077977.759795:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077977.759803:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077977.759841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612761216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077977.759875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077977.759880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077977.759882:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077977.759887:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4965 00000100:00080000:3.0:1769077977.759891:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077977.759897:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077977.759907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077977.759920:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612761216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769077977.759922:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077977.759933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077977.759936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077977.759937:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077977.759940:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4965 00000100:00080000:3.0:1769077977.759943:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077977.759946:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077977.759967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077977.759978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612761472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077977.759985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077977.759988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077977.759989:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077977.759991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4965 00000100:00080000:3.0:1769077977.759994:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077977.759999:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769077977.760000:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077977.760005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077977.760014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612761600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077977.760035:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077977] req@ffff8b10037dc000 x1855007612761216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077977.760050:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612761216/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077977.760063:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077977.760066:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077977.760069:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612761216:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077977.760087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761344:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077977.760109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077977.760122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077977] req@ffff8b10037dd500 x1855007612761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077977.760135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612761344/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077977.760143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077977.760146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077977.760150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761344:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077977.760156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077977.760177:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077977.760188:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077977] req@ffff8b10037ddf80 x1855007612761472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077977.760211:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612761472/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077977.760219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077977.760222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077977.760225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077977.760232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077977.760252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077977.760265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077977] req@ffff8b10037de300 x1855007612761600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077977.760276:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612761600/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077977.760285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077977.760288:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077977.760292:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077978.655782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077978.655827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077979.679762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077979.679824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077980.703766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077980.703809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769077981.727904:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077981.727942:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077982.751763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077982.751908:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077982.879784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077982.879799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077982.879805:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077982.879818:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4970 00000100:00080000:3.0:1769077982.879832:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077982.879840:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769077982.879908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612761728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077982.879935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077982.879939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077982.879941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077982.879945:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4970 00000100:00080000:3.0:1769077982.879949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077982.879954:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077982.879965:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077982.879980:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077982.879991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077982.879993:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612761728:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769077982.879995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077982.879998:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077982.880006:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4970 00000100:00080000:3.0:1769077982.880010:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077982.880015:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077982.880020:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077982.880033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612761984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077982.880042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077982.880046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077982.880047:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077982.880051:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4970 00000100:00080000:3.0:1769077982.880055:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077982.880059:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769077982.880063:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077982.880073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612762112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077982.880087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077982.880134:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff8b10037ded80 x1855007612761728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077982.880154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612761728/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077982.880165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077982.880170:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077982.880174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612761728:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077982.880202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761856:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077982.880228:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077982.880250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff8b10037de680 x1855007612761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077982.880264:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612761856/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077982.880272:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077982.880276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077982.880279:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761856:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077982.880285:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077982.880306:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077982.880316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff8b10037dc380 x1855007612761984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077982.880328:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612761984/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077982.880336:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077982.880339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077982.880343:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612761984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077982.880359:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077982.880379:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077982.880390:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff8b10037dd180 x1855007612762112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077982.880409:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612762112/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077982.880417:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077982.880420:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077982.880423:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077983.775756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077983.775821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077984.799759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077984.799805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077985.823739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077985.823798:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077986.847717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077986.847861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077987.871742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077987.871922:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077987.999773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.999809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077987.999814:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.999826:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4976 00000100:00080000:2.0:1769077987.999841:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.999849:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769077987.999896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612762240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077987.999931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.999936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077987.999938:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.999943:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4976 00000100:00080000:2.0:1769077987.999948:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.999959:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077987.999971:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077987.999985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612762368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077987.999992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.999995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077987.999997:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077988.000000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4976 00000100:00080000:2.0:1769077988.000003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077988.000006:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077988.000009:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077988.000015:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612762240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769077988.000022:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612762496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077988.000035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077988.000042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077988.000044:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077988.000046:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4976 00000100:00080000:2.0:1769077988.000048:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077988.000050:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769077988.000053:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077988.000063:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612762624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077988.000106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077988.000147:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077988] req@ffff8b1005d5bb80 x1855007612762240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077988.000161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612762240/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077988.000169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077988.000173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077988.000175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612762240:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077988.000181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762368:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077988.000198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077988.000206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077988] req@ffff8b1005d59880 x1855007612762368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077988.000213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612762368/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077988.000219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077988.000230:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077988.000233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762368:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077988.000254:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077988.000267:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077988.000278:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077988] req@ffff8b1005d5aa00 x1855007612762496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077988.000286:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612762496/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077988.000291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077988.000293:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077988.000295:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077988.000299:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077988.000310:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077988.000316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077988] req@ffff8b1005d5b100 x1855007612762624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077988.000323:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612762624/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077988.000327:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077988.000329:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077988.000331:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077988.895787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077988.895874:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077989.919739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077989.919816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077990.943800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077990.943855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077991.967761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077991.967812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077992.991759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077992.991846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077993.119802:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.119819:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077993.119824:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.119835:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4981 00000100:00080000:1.0:1769077993.119850:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.119859:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077993.119901:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612762752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.119927:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.119930:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077993.119931:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.119940:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4981 00000100:00080000:1.0:1769077993.119944:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.119947:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077993.119979:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077993.119985:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612762752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769077993.119993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612762880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.120003:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.120006:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077993.120009:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.120013:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4981 00000100:00080000:1.0:1769077993.120017:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.120021:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077993.120026:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077993.120039:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612763008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.120048:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.120052:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077993.120054:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.120059:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4981 00000100:00080000:1.0:1769077993.120063:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.120066:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769077993.120072:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769077993.120074:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077993.120088:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612763136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077993.120122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff8b1006135c00 x1855007612762752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077993.120143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612762752/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077993.120154:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077993.120159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077993.120163:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612762752:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077993.120189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762880:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077993.120207:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077993.120219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff8b1006137b80 x1855007612762880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077993.120228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612762880/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077993.120233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077993.120236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077993.120238:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612762880:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077993.120241:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077993.120259:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077993.120269:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff8b1006135880 x1855007612763008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077993.120280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612763008/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077993.120289:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077993.120292:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077993.120296:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077993.120301:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077993.120318:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077993.120328:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff8b1006135500 x1855007612763136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077993.120339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612763136/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077993.120346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077993.120349:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077993.120352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077994.015739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077994.015804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077995.039801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077995.039972:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077996.063892:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077996.065519:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077997.087736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077997.087793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077998.111707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077998.111747:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077998.239755:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.239763:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077998.239766:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.239774:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4986 00000100:00080000:1.0:1769077998.239783:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.239788:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769077998.239817:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612763264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.239838:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.239841:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077998.239843:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.239847:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4986 00000100:00080000:1.0:1769077998.239850:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.239853:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077998.239861:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077998.239873:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612763392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.239881:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.239884:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077998.239885:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.239888:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4986 00000100:00080000:1.0:1769077998.239891:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077998.239892:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612763264:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769077998.239895:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769077998.239899:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077998.239907:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612763520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.239915:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.239917:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077998.239919:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.239921:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4986 00000100:00080000:1.0:1769077998.239935:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.239939:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769077998.239943:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769077998.239943:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077998.239951:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612763648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077998.239972:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff8b1006136d80 x1855007612763264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077998.239988:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612763264/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077998.239997:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077998.240001:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077998.240004:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612763264:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769077998.240009:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763392:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769077998.240024:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077998.240032:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff8b1006136680 x1855007612763392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077998.240040:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612763392/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077998.240044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077998.240046:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077998.240048:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763392:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769077998.240063:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077998.240073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077998.240078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff8b1006137100 x1855007612763520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077998.240084:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612763520/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077998.240093:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077998.240095:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077998.240097:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769077998.240099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769077998.240114:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769077998.240119:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff8b1006134e00 x1855007612763648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077998.240126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612763648/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077998.240130:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077998.240132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769077998.240133:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769077999.135723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077999.135771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078000.159730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078000.159793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078001.183769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078001.183840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078002.207784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078002.207858:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078003.231749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078003.231809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078003.359766:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.359775:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078003.359779:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.359787:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4991 00000100:00080000:1.0:1769078003.359796:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.359803:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078003.359832:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612763776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078003.359854:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.359857:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078003.359859:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.359863:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4991 00000100:00080000:1.0:1769078003.359868:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.359871:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078003.359879:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078003.359887:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612763776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078003.359894:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078003.359904:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.359907:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078003.359908:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.359912:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4991 00000100:00080000:1.0:1769078003.359916:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.359921:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078003.359926:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078003.359942:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612764032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078003.359950:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769078003.359960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.359965:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078003.359967:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.359973:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4991 00000100:00080000:1.0:1769078003.359978:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.359982:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078003.359986:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff8b1006134700 x1855007612763776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078003.359988:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078003.360005:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612764160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078003.360009:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612763776/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078003.360035:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078003.360041:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078003.360046:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612763776:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078003.360066:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763904:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078003.360080:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078003.360089:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff8b1006137480 x1855007612763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078003.360097:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612763904/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078003.360103:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078003.360105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078003.360107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612763904:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078003.360112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764032:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078003.360124:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078003.360130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff8b1006134a80 x1855007612764032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078003.360138:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612764032/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078003.360144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078003.360146:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078003.360148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078003.360151:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078003.360164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078003.360169:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff8b1006134e00 x1855007612764160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078003.360178:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612764160/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078003.360183:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078003.360185:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078003.360187:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078004.255757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078004.255846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078005.279745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078005.279836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078006.303765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078006.303780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078007.327741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078007.327815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078008.351778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078008.351817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078008.479716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.479727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078008.479732:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.479743:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 4996 00000100:00080000:2.0:1769078008.479755:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.479762:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078008.479803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612764288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.479827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.479830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078008.479831:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.479834:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4996 00000100:00080000:2.0:1769078008.479836:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.479840:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078008.479849:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078008.479857:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.479865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.479868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078008.479870:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078008.479873:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612764288:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078008.479875:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4996 00000100:00080000:2.0:1769078008.479880:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.479883:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078008.479885:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078008.479894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612764544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.479900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.479902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078008.479903:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.479906:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 4996 00000100:00080000:2.0:1769078008.479914:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.479916:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078008.479920:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078008.479926:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612764672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078008.479947:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078008.479983:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff8b1005d58380 x1855007612764288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078008.479998:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612764288/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078008.480006:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078008.480009:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078008.480012:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612764288:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078008.480033:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764416:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078008.480048:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078008.480056:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff8b1005d5ad80 x1855007612764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078008.480070:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612764416/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078008.480078:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078008.480081:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078008.480084:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764416:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078008.480090:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078008.480111:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078008.480121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff8b1005d5b100 x1855007612764544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078008.480130:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612764544/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078008.480135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078008.480137:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078008.480139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078008.480143:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078008.480155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078008.480161:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff8b1005d5aa00 x1855007612764672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078008.480168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612764672/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078008.480177:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078008.480179:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078008.480181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078009.375729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078009.375813:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078010.399755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078010.399830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078011.423771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078011.423838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078012.447747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078012.447815:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078013.471755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078013.471849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078013.599872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078013.599884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078013.599891:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078013.599903:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5001 00000100:00080000:0.0:1769078013.599917:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078013.599926:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078013.599974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612764800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078013.600018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078013.600024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078013.600027:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078013.600033:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5001 00000100:00080000:0.0:1769078013.600039:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078013.600044:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078013.600056:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078013.600073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612764928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078013.600096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078013.600101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078013.600104:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078013.600108:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5001 00000100:00080000:0.0:1769078013.600112:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078013.600117:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078013.600125:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078013.600134:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612764800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078013.600145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612765056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078013.600162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078013.600166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078013.600168:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078013.600173:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5001 00000100:00080000:0.0:1769078013.600177:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078013.600181:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078013.600187:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078013.600203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612765184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769078013.600231:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078013.600289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff8b1006025880 x1855007612764800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078013.600315:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612764800/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078013.600329:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078013.600334:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078013.600339:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612764800:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078013.600381:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764928:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078013.600418:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078013.600436:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff8b1006024000 x1855007612764928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078013.600452:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612764928/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078013.600463:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078013.600467:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078013.600472:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612764928:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078013.600501:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078013.600524:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078013.600536:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff8b1006026a00 x1855007612765056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078013.600551:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612765056/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078013.600561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078013.600565:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078013.600568:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078013.600576:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765184:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078013.600594:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078013.600606:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff8b1006026300 x1855007612765184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078013.600629:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612765184/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078013.600677:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078013.600681:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078013.600685:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078014.495767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078014.495820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078015.519701:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078015.519785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078016.543782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078016.543834:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078017.567757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078017.567839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078018.591766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078018.591876:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078018.719763:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.719776:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078018.719781:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.719792:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5006 00000100:00080000:1.0:1769078018.719805:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.719812:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078018.719859:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612765312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.719888:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.719892:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078018.719893:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.719897:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5006 00000100:00080000:1.0:1769078018.719899:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.719902:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078018.719910:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078018.719919:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612765440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.719926:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.719930:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078018.719932:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.719936:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5006 00000100:00080000:1.0:1769078018.719939:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.719943:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078018.719949:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078018.719957:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612765568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.719964:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.719967:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078018.719968:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.719970:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5006 00000100:00080000:1.0:1769078018.719972:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.719974:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078018.719977:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078018.719984:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612765696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078018.720041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612765312:192.168.204.102@tcp:250:kworker.0 00000800:00000100:3.0:1769078018.720223:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078018.720368:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff8b1006137100 x1855007612765312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078018.720426:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612765312/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078018.720439:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078018.720460:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078018.720465:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612765312:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078018.720474:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765440:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078018.720603:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078018.720704:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff8b1006136680 x1855007612765440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078018.720803:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612765440/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078018.720816:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078018.720822:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078018.720827:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765440:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078018.720838:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078018.720897:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078018.720917:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff8b1006136d80 x1855007612765568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078018.720925:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612765568/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078018.720931:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078018.720933:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078018.720936:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078018.720940:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078018.720951:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078018.720958:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff8b1006135500 x1855007612765696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078018.720965:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612765696/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078018.720975:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078018.720977:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078018.720979:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078019.615759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078019.615815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078020.639767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078020.639839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078021.663759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078021.663856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078022.687768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078022.687852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078023.711786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078023.711842:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078023.839764:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078023.839780:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078023.839786:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078023.839801:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5011 00000100:00080000:2.0:1769078023.839817:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078023.839827:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078023.839878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612765824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078023.839906:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078023.839909:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078023.839911:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078023.839914:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5011 00000100:00080000:2.0:1769078023.839917:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078023.839921:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078023.839930:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078023.839943:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612765952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078023.839952:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078023.839956:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769078023.839956:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612765824:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078023.839960:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078023.839968:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5011 00000100:00080000:2.0:1769078023.839970:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078023.839973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078023.839976:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078023.839986:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612766080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078023.839993:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078023.839996:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078023.839997:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078023.839999:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5011 00000100:00080000:2.0:1769078023.840001:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078023.840003:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078023.840006:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078023.840013:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612766208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769078023.840039:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078023.840079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff8b1005d59880 x1855007612765824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078023.840095:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612765824/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078023.840104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078023.840108:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078023.840111:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612765824:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078023.840138:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765952:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078023.840164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078023.840192:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff8b1005d5bb80 x1855007612765952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078023.840208:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612765952/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078023.840215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078023.840218:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078023.840220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612765952:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078023.840225:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078023.840242:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078023.840249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff8b1005d5b480 x1855007612766080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078023.840257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612766080/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078023.840263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078023.840265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078023.840267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078023.840270:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078023.840281:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078023.840288:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff8b1005d59500 x1855007612766208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078023.840296:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612766208/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078023.840301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078023.840303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078023.840305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078024.735743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078024.735818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078025.759873:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078025.759910:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078026.783735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078026.783829:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078027.807831:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078027.807855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078028.831740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078028.831819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078028.959742:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078028.959751:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078028.959755:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078028.959762:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5017 00000100:00080000:2.0:1769078028.959772:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078028.959778:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078028.959815:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612766336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078028.959837:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078028.959840:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078028.959841:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078028.959844:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5017 00000100:00080000:2.0:1769078028.959847:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078028.959849:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078028.959856:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078028.959865:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078028.959873:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078028.959875:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078028.959876:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078028.959879:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5017 00000100:00080000:2.0:1769078028.959881:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078028.959884:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078028.959887:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078028.959893:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612766336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078028.959896:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612766592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078028.959908:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078028.959910:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078028.959912:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078028.959915:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5017 00000100:00080000:2.0:1769078028.959917:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078028.959919:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078028.959922:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078028.959930:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612766720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078028.959994:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078028.960031:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078028.960056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078029.855767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078029.855848:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078030.879805:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078030.879808:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078031.903741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078031.903812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769078032.031932:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769078032.031976:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769078032.031994:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078032.032067:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078032.032084:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078032.032094:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:3.0:1769078032.032204:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078032] req@ffff8b1005d5b800 x1855007612766592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078032.032233:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612766592/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078032.032247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078032.032253:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078032.032257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078032.032264:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078032] req@ffff8b1005d58a80 x1855007612766720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078032.032277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612766720/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078032.032284:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078032.032287:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078032.032290:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078032.032297:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078032] req@ffff8b1005d58e00 x1855007612766336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078032.032309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612766336/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078032.032322:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078032.032325:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078032.032329:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612766336:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078032.032336:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078032] req@ffff8b1005d5a300 x1855007612766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078032.032348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612766464/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078032.032355:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078032.032359:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078032.032362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766464:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078032.927745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078032.927866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078033.951792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078033.951859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078034.079755:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078034.079766:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078034.079770:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078034.079779:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5022 00000100:00080000:3.0:1769078034.079792:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078034.079801:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078034.079852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612766848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078034.079893:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078034.079897:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078034.079899:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078034.079906:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5022 00000100:00080000:3.0:1769078034.079911:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078034.079916:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078034.079927:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078034.079943:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078034.079955:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078034.079959:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078034.079962:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078034.079970:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5022 00000100:00100000:0.0:1769078034.079976:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612766848:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078034.079977:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078034.079987:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078034.079997:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078034.080029:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612767104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078034.080040:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078034.080044:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078034.080046:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078034.080050:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5022 00000100:00080000:3.0:1769078034.080057:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078034.080061:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078034.080067:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078034.080080:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612767232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078034.080091:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078034.080162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff8b10037dca80 x1855007612766848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078034.080188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612766848/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078034.080200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078034.080204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078034.080208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612766848:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078034.080242:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766976:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078034.080272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078034.080286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff8b10037dd180 x1855007612766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078034.080297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612766976/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078034.080305:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078034.080308:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078034.080311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612766976:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078034.080322:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078034.080342:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078034.080353:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff8b10037dc380 x1855007612767104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078034.080364:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612767104/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078034.080371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078034.080374:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078034.080377:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078034.080384:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078034.080403:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078034.080414:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff8b10037de680 x1855007612767232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078034.080424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612767232/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078034.080431:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078034.080434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078034.080437:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078034.975752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078034.975830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078035.999752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078035.999819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078037.023707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078037.023778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078038.047777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078038.047783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078039.071753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078039.071813:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078039.199771:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078039.199783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078039.199787:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078039.199798:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5027 00000100:00080000:0.0:1769078039.199807:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078039.199814:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078039.199849:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612767360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078039.199875:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078039.199878:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078039.199880:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078039.199885:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5027 00000100:00080000:0.0:1769078039.199889:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078039.199893:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078039.199902:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078039.199906:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612767360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078039.199930:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612767488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078039.199942:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078039.199945:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078039.199948:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078039.199952:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5027 00000100:00080000:0.0:1769078039.199969:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078039.199973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769078039.199978:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769078039.199981:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078039.199996:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612767616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078039.200006:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff8b1006026d80 x1855007612767360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078039.200007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078039.200016:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078039.200021:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078039.200026:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5027 00000100:00100000:1.0:1769078039.200029:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612767360/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078039.200031:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078039.200039:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:1.0:1769078039.200042:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769078039.200045:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1769078039.200048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078039.200054:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612767360:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078039.200059:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612767744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078039.200082:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767488:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078039.200106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078039.200124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff8b1006027b80 x1855007612767488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078039.200141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612767488/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078039.200150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078039.200153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078039.200156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767488:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078039.200162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078039.200183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078039.200196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff8b1006025f80 x1855007612767616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078039.200208:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612767616/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078039.200215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078039.200217:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078039.200219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078039.200223:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078039.200237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078039.200244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff8b1006024a80 x1855007612767744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078039.200251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612767744/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078039.200256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078039.200258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078039.200260:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612767744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078040.095730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078040.095822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078041.119749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078041.119807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078042.143951:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078042.144307:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078043.167750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078043.167805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078044.191739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078044.191799:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078044.319777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078044.319791:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078044.319796:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078044.319808:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5032 00000100:00080000:0.0:1769078044.319822:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078044.319844:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078044.319890:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612767872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078044.319919:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078044.319922:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078044.319925:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078044.319931:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5032 00000100:00080000:0.0:1769078044.319935:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078044.319939:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078044.319957:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078044.319971:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612768000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078044.319982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612767872:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078044.319990:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078044.319995:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078044.319998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078044.320005:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5032 00000100:00080000:0.0:1769078044.320010:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078044.320016:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078044.320023:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078044.320037:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612768128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078044.320049:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078044.320060:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078044.320065:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078044.320068:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078044.320076:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5032 00000100:00080000:0.0:1769078044.320082:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078044.320084:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078044] req@ffff8b1006027480 x1855007612767872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078044.320092:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078044.320108:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078044.320114:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612767872/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078044.320124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769078044.320130:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612768256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078044.320132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078044.320142:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612767872:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078044.320173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768000:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078044.320189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078044.320196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078044] req@ffff8b1006024700 x1855007612768000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078044.320203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612768000/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078044.320212:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078044.320215:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078044.320217:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768000:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078044.320221:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078044.320237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078044.320247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078044] req@ffff8b1006024e00 x1855007612768128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078044.320259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612768128/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078044.320266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078044.320269:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078044.320272:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078044.320278:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078044.320296:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078044.320304:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078044] req@ffff8b1006024380 x1855007612768256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078044.320313:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612768256/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078044.320321:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078044.320324:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078044.320327:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078045.215847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078045.215867:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078046.239767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078046.239833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078047.263765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078047.263838:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078048.287729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078048.287795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078049.311800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078049.311848:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078049.439799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.439814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078049.439819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.439829:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5037 00000100:00080000:3.0:1769078049.439841:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.439848:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078049.439890:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612768384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.439920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.439923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078049.439924:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.439927:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5037 00000100:00080000:3.0:1769078049.439930:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.439932:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078049.439941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078049.439950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612768512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.439958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.439960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078049.439961:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.439964:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5037 00000100:00080000:3.0:1769078049.439966:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.439969:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078049.439973:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078049.439979:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612768640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.439986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.439988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078049.439989:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.439991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5037 00000100:00080000:3.0:1769078049.439994:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.439997:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078049.440000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078049.440005:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612768768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078049.440024:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612768384:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078049.440090:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078049.440127:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff8b10037ded80 x1855007612768384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078049.440146:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612768384/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078049.440157:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078049.440162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078049.440166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612768384:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078049.440173:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768512:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078049.440194:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078049.440205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff8b10037de300 x1855007612768512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078049.440217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612768512/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078049.440226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078049.440229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078049.440232:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768512:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078049.440238:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078049.440252:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078049.440261:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff8b10037ddf80 x1855007612768640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078049.440272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612768640/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078049.440279:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078049.440282:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078049.440285:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078049.440309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078049.440327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078049.440337:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff8b10037dd500 x1855007612768768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078049.440348:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612768768/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078049.440356:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078049.440359:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078049.440362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612768768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078050.335700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078050.335785:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078051.359751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078051.359810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078052.383780:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078052.383792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078053.407842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078053.407865:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078054.431717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078054.431792:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078054.559772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078054.559781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078054.559792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078054.559803:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5042 00000100:00080000:0.0:1769078054.559813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078054.559819:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078054.559853:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612768896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078054.559876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078054.559879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078054.559880:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078054.559883:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5042 00000100:00080000:0.0:1769078054.559885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078054.559888:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078054.559894:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078054.559903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078054.559910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078054.559914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078054.559915:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078054.559918:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5042 00000100:00080000:0.0:1769078054.559921:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078054.559924:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612768896:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078054.559925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078054.559933:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078054.559940:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612769152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078054.559948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078054.559950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078054.559951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078054.559954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5042 00000100:00080000:0.0:1769078054.559956:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078054.559958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078054.559962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078054.559969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612769280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078054.559994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078054.560023:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff8b1006024380 x1855007612768896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078054.560036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612768896/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078054.560044:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078054.560048:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078054.560051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612768896:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078054.560056:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769024:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078054.560071:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078054.560079:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff8b1006024e00 x1855007612769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078054.560093:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612769024/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078054.560102:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078054.560105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078054.560109:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769024:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078054.560132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078054.560151:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078054.560159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff8b1006024700 x1855007612769152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078054.560167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612769152/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078054.560173:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078054.560175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078054.560178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078054.560182:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078054.560193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078054.560199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff8b1006027480 x1855007612769280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078054.560206:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612769280/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078054.560211:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078054.560213:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078054.560215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078055.455764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078055.455831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078056.479775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078056.479865:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078057.503794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078057.503873:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078058.527773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078058.527830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769078059.551852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078059.551859:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078059.679760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.679773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078059.679777:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.679787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5047 00000100:00080000:3.0:1769078059.679799:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.679807:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078059.679858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612769408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.679883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.679886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078059.679888:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.679891:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5047 00000100:00080000:3.0:1769078059.679894:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.679896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078059.679905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078059.679914:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.679923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.679926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078059.679928:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.679931:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5047 00000100:00080000:3.0:1769078059.679933:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.679939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078059.679943:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078059.679951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612769664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.679959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.679961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078059.679963:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.679966:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5047 00000100:00080000:3.0:1769078059.679968:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.679971:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078059.679974:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078059.679983:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612769408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078059.679983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612769792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078059.680081:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078059.680125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff8b10037dc000 x1855007612769408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078059.680145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612769408/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078059.680155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078059.680158:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078059.680162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612769408:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078059.680168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769536:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078059.680187:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078059.680196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff8b10037dd880 x1855007612769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078059.680205:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612769536/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078059.680210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078059.680212:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078059.680214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769536:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078059.680218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078059.680230:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078059.680236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff8b10037df100 x1855007612769664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078059.680243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612769664/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078059.680248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078059.680251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078059.680252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078059.680273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078059.680284:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078059.680290:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff8b10037ddc00 x1855007612769792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078059.680298:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612769792/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078059.680308:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078059.680310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078059.680312:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612769792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078060.575763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078060.575827:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078061.599758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078061.599809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078062.623691:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078062.623778:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078063.647771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078063.647847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078064.671763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078064.671836:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078064.799741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078064.799758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078064.799764:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078064.799778:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5052 00000100:00080000:3.0:1769078064.799798:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078064.799805:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078064.799852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612769920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078064.799881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078064.799884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078064.799886:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078064.799889:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5052 00000100:00080000:3.0:1769078064.799893:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078064.799896:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078064.799905:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078064.799916:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612770048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078064.799924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078064.799927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078064.799928:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078064.799931:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5052 00000100:00080000:3.0:1769078064.799933:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078064.799936:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078064.799939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078064.799951:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612769920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078064.799952:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612770176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078064.799965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078064.799968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078064.799970:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078064.799973:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5052 00000100:00080000:3.0:1769078064.799975:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078064.799978:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078064.799988:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078064.799999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612770304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078064.800033:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078064.800072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff8b10037df800 x1855007612769920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078064.800088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612769920/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078064.800097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078064.800101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078064.800104:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612769920:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078064.800110:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770048:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078064.800125:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078064.800134:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff8b10037dc700 x1855007612770048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078064.800142:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612770048/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078064.800148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078064.800150:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078064.800152:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770048:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078064.800176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078064.800189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078064.800196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff8b10037dce00 x1855007612770176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078064.800204:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612770176/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078064.800210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078064.800212:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078064.800214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078064.800218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770304:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078064.800228:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078064.800234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff8b10037df480 x1855007612770304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078064.800242:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612770304/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078064.800247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078064.800249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078064.800251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770304:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078065.695719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078065.695813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078066.719850:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078066.719862:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078067.743712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078067.743826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078068.767760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078068.767852:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078069.791764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078069.791833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078069.919772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078069.919789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078069.919793:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078069.919801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5058 00000100:00080000:3.0:1769078069.919811:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078069.919817:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078069.919851:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612770432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078069.919877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078069.919880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078069.919881:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078069.919884:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5058 00000100:00080000:3.0:1769078069.919886:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078069.919889:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078069.919895:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078069.919909:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612770560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078069.919922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078069.919927:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612770432:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078069.919927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078069.919932:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078069.919939:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5058 00000100:00080000:3.0:1769078069.919944:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078069.919948:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078069.919955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078069.919969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612770688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078069.919976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078069.919979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078069.919980:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078069.919984:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5058 00000800:00000100:1.0:1769078069.919989:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078069.919989:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078069.919998:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078069.920005:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078069.920019:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612770816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078069.920033:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff8b10037dea00 x1855007612770432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078069.920051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612770432/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078069.920060:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078069.920064:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078069.920068:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612770432:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078069.920089:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770560:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078069.920113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078069.920127:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff8b10037df480 x1855007612770560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078069.920137:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612770560/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078069.920143:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078069.920145:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078069.920148:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770560:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078069.920152:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770688:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078069.920167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078069.920174:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff8b10037dce00 x1855007612770688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078069.920182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612770688/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078069.920191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078069.920194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078069.920196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078069.920199:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770816:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078069.920209:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078069.920215:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff8b10037dc700 x1855007612770816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078069.920222:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612770816/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078069.920228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078069.920229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078069.920231:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612770816:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078070.815759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078070.815855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078071.839731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078071.839843:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078072.863772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078072.863858:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078073.887800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078073.887803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078074.911750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078074.911812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078075.039785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078075.039791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078075.039796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078075.039804:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5063 00000100:00080000:0.0:1769078075.039814:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078075.039820:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078075.039849:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612770944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078075.039869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078075.039873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078075.039874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078075.039878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5063 00000100:00080000:0.0:1769078075.039881:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078075.039885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078075.039892:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078075.039903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612771072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078075.039909:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612770944:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078075.039913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078075.039917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078075.039919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078075.039923:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5063 00000100:00080000:0.0:1769078075.039926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078075.039929:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078075.039934:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078075.039944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612771200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078075.039951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078075.039954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078075.039955:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078075.039958:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5063 00000100:00080000:0.0:1769078075.039962:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078075.039965:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078075.039968:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078075.039972:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078075.039978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612771328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078075.040007:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff8b1006024a80 x1855007612770944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078075.040028:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612770944/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078075.040041:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078075.040047:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078075.040051:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612770944:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078075.040079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771072:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078075.040109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078075.040124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff8b1006025f80 x1855007612771072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078075.040140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612771072/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078075.040149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078075.040155:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078075.040160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771072:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078075.040168:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078075.040190:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078075.040203:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff8b1006027b80 x1855007612771200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078075.040216:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612771200/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078075.040225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078075.040229:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078075.040233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078075.040239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771328:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078075.040260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078075.040274:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078075/real 1769078075] req@ffff8b1006026d80 x1855007612771328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078075.040288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612771328/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078075.040298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078075.040302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078075.040305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771328:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078075.935693:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078075.935775:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078076.959775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078076.959839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078077.983723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078077.983862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078079.007783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078079.007847:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078080.031753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078080.031803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078080.159791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.159805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078080.159811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.159822:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5068 00000100:00080000:3.0:1769078080.159837:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.159844:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078080.159892:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612771456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.159923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.159927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078080.159929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.159934:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5068 00000100:00080000:3.0:1769078080.159937:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.159942:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078080.159952:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078080.159965:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.159975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.159979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078080.159981:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.159985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5068 00000100:00080000:3.0:1769078080.159989:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.159993:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078080.159998:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078080.160010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612771712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078080.160012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612771456:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078080.160028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.160031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078080.160034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.160038:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5068 00000100:00080000:3.0:1769078080.160043:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.160047:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078080.160052:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078080.160064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612771840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078080.160087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078080.160121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff8b10037df800 x1855007612771456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078080.160135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612771456/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078080.160144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078080.160147:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078080.160150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612771456:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078080.160155:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771584:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078080.160168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078080.160176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff8b10037ddc00 x1855007612771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078080.160183:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612771584/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078080.160192:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078080.160194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078080.160196:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771584:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078080.160217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078080.160239:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078080.160248:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff8b10037df100 x1855007612771712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078080.160255:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612771712/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078080.160260:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078080.160263:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078080.160264:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078080.160268:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771840:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078080.160279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078080.160285:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff8b10037dd880 x1855007612771840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078080.160292:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612771840/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078080.160302:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078080.160304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078080.160306:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612771840:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078081.055815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078081.055857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078082.079773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078082.079883:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078083.103729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078083.103797:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078084.127751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078084.127825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078085.151734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078085.151796:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078085.279776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078085.279791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078085.279796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078085.279806:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5073 00000100:00080000:3.0:1769078085.279824:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078085.279831:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078085.279871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612771968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078085.279897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078085.279900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078085.279901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078085.279904:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5073 00000100:00080000:3.0:1769078085.279906:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078085.279908:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078085.279916:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078085.279925:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078085.279937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078085.279945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769078085.279946:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612771968:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078085.279947:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078085.279955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5073 00000100:00080000:3.0:1769078085.279958:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078085.279960:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078085.279963:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078085.279972:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612772224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078085.279979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078085.279981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078085.279982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078085.279985:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5073 00000100:00080000:3.0:1769078085.279987:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078085.279990:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078085.279993:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078085.279999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612772352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078085.280015:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078085.280051:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff8b10037dc000 x1855007612771968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078085.280065:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612771968/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078085.280072:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078085.280076:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078085.280079:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612771968:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078085.280099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772096:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078085.280113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078085.280121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff8b10037dd500 x1855007612772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078085.280128:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612772096/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078085.280133:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078085.280135:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078085.280137:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772096:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078085.280140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078085.280168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078085.280176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff8b10037ddf80 x1855007612772224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078085.280183:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612772224/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078085.280189:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078085.280190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078085.280192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078085.280196:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772352:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078085.280205:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078085.280211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff8b10037de300 x1855007612772352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078085.280218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612772352/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078085.280223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078085.280224:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078085.280226:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772352:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078086.175742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078086.175835:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078087.199714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078087.199812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078088.223864:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078088.223891:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078089.247698:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078089.247801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078090.271742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078090.271805:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078090.399735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.399755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078090.399761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.399773:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5078 00000100:00080000:3.0:1769078090.399784:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.399792:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078090.399829:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612772480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078090.399864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.399874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078090.399876:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.399881:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5078 00000100:00080000:3.0:1769078090.399885:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.399891:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078090.399901:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078090.399913:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612772480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078090.399919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612772608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078090.399935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.399939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078090.399941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.399945:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5078 00000100:00080000:3.0:1769078090.399951:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.399955:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078090.399961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078090.399970:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078090.399976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612772736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078090.399995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.399999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769078090.400002:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff8b10037ded80 x1855007612772480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078090.400004:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.400016:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5078 00000100:00080000:3.0:1769078090.400022:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078090.400023:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612772480/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078090.400032:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00080000:1.0:1769078090.400038:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769078090.400042:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1769078090.400044:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078090.400050:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612772480:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078090.400076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612772864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078090.400087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772608:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078090.400129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078090.400149:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff8b10037de680 x1855007612772608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078090.400164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612772608/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078090.400172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078090.400176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078090.400180:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772608:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078090.400185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078090.400203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078090.400211:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff8b10037dc380 x1855007612772736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078090.400220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612772736/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078090.400225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078090.400228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078090.400230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078090.400239:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772864:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078090.400251:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078090.400257:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff8b10037dd180 x1855007612772864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078090.400265:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612772864/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078090.400270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078090.400272:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078090.400275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612772864:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078091.295738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078091.295805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078092.319751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078092.319831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078093.343704:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078093.343770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078094.367760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078094.367878:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078095.391823:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078095.391858:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078095.519796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078095.519809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078095.519814:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078095.519824:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5083 00000100:00080000:3.0:1769078095.519838:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078095.519847:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078095.519893:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612772992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078095.519920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078095.519924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078095.519927:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078095.519935:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5083 00000100:00080000:3.0:1769078095.519939:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078095.519944:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078095.519954:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612772992:192.168.204.102@tcp:250:kworker.0 00000008:00100000:3.0:1769078095.519958:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078095.520002:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612773120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078095.520012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078095.520015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078095.520016:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078095.520020:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5083 00000100:00080000:3.0:1769078095.520023:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078095.520026:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078095.520031:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078095.520044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612773248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078095.520055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078095.520059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078095.520061:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078095.520067:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5083 00000100:00080000:3.0:1769078095.520074:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078095.520081:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078095.520087:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078095.520092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078095.520107:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612773376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078095.520132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078095.520176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078096.415753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078096.415825:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078097.439787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078097.439845:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078098.463795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078098.463850:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769078098.592095:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769078098.592158:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078098.592183:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078098.592245:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078098.592256:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078098.592266:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769078098.592477:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078098] req@ffff8b10037de680 x1855007612773376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078098.592510:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612773376/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078098.592526:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078098.592533:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078098.592538:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773376:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078098.592549:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078098] req@ffff8b10037dd180 x1855007612773120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078098.592563:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612773120/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078098.592580:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078098.592585:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078098.592589:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773120:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078098.592597:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078098] req@ffff8b10037dc380 x1855007612773248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078098.592609:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612773248/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078098.592618:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078098.592623:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078098.592627:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078098.592635:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078098] req@ffff8b10037dca80 x1855007612772992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078098.593524:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612772992/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078098.593537:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078098.593549:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078098.593553:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612772992:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078099.487758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078099.487837:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078100.511765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078100.511857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078100.639754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.639766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078100.639770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.639787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5088 00000100:00080000:3.0:1769078100.639798:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.639804:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078100.639839:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612773504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.639867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.639872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078100.639874:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.639878:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5088 00000100:00080000:3.0:1769078100.639882:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.639898:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078100.639910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078100.639932:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612773632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.639945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078100.639950:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612773504:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078100.639951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078100.639956:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.639968:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5088 00000100:00080000:3.0:1769078100.639973:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.639977:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078100.639984:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078100.640008:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612773760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.640017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.640021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078100.640023:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.640028:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5088 00000100:00080000:3.0:1769078100.640033:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.640037:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078100.640042:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078100.640053:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612773888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078100.640071:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078100.640130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff8b10037ded80 x1855007612773504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078100.640156:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612773504/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078100.640170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078100.640176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078100.640181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612773504:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078100.640218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773632:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078100.640248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078100.640265:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff8b10037de300 x1855007612773632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078100.640279:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612773632/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078100.640291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078100.640296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078100.640301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773632:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078100.640311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078100.640349:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078100.640362:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff8b10037ddf80 x1855007612773760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078100.640379:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612773760/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078100.640388:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078100.640392:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078100.640396:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773760:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078100.640404:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773888:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078100.640429:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078100.640441:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff8b10037dd500 x1855007612773888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078100.640455:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612773888/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078100.640465:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078100.640469:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078100.640472:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612773888:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078101.535770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078101.535884:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078102.559848:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078102.559898:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078103.583795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078103.583899:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078104.607748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078104.607855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078105.631733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078105.631827:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078105.759784:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078105.759812:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078105.759818:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078105.759831:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5093 00000100:00080000:1.0:1769078105.759846:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078105.759855:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078105.759904:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612774016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078105.759948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078105.759953:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078105.759955:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078105.759961:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5093 00000100:00080000:1.0:1769078105.759965:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078105.759971:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078105.759983:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078105.759993:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612774016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078105.759999:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078105.760013:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078105.760017:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078105.760020:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078105.760024:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5093 00000100:00080000:1.0:1769078105.760027:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078105.760031:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078105.760037:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078105.760055:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612774272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078105.760067:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078105.760072:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1769078105.760073:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769078105.760075:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078105.760082:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5093 00000100:00080000:1.0:1769078105.760088:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078105.760093:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078105.760098:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078105.760110:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612774400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078105.760116:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff8b1006135880 x1855007612774016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078105.760144:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612774016/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078105.760153:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078105.760157:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078105.760160:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612774016:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078105.760183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774144:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078105.760200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078105.760209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff8b1006137b80 x1855007612774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078105.760218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612774144/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078105.760224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078105.760226:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078105.760228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774144:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078105.760232:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078105.760244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078105.760250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff8b1006135c00 x1855007612774272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078105.760258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612774272/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078105.760268:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078105.760270:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078105.760273:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078105.760276:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774400:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078105.760287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078105.760293:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff8b1006136a00 x1855007612774400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078105.760301:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612774400/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078105.760307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078105.760308:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078105.760310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774400:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078106.655706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078106.655853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078107.679742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078107.679828:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078108.703781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078108.703863:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078109.727832:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078109.727838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078110.751771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078110.751830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078110.879772:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.879783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078110.879787:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.879795:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5098 00000100:00080000:1.0:1769078110.879805:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.879811:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078110.879841:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612774528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.879869:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.879873:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078110.879874:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.879877:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5098 00000100:00080000:1.0:1769078110.879879:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.879882:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078110.879889:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078110.879900:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612774656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.879908:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.879913:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078110.879915:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.879918:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5098 00000100:00080000:1.0:1769078110.879920:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.879923:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078110.879926:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078110.879930:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612774528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078110.879939:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612774784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.879948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.879950:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078110.879952:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.879956:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5098 00000100:00080000:1.0:1769078110.879958:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.879961:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078110.879974:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078110.879988:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612774912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769078110.880007:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078110.880046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078110] req@ffff8b1006136300 x1855007612774528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078110.880150:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612774528/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078110.880163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078110.880168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078110.880172:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612774528:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078110.880180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774656:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078110.880218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078110.880352:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078110] req@ffff8b1006137800 x1855007612774656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078110.880422:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612774656/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078110.880432:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078110.880436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078110.880440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774656:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078110.880465:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078110.880490:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078110.880502:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078110] req@ffff8b1006135180 x1855007612774784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078110.880516:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612774784/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078110.880525:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078110.880528:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078110.880532:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078110.880539:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774912:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078110.880557:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078110.880568:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078110] req@ffff8b1006136a00 x1855007612774912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078110.880582:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612774912/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078110.880591:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078110.880595:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078110.880599:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612774912:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078111.775750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078111.775814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078112.799732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078112.799851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078113.823762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078113.823833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078114.847781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078114.847817:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078115.871781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078115.871884:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078115.999796:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.999809:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078115.999814:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.999826:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5104 00000100:00080000:0.0:1769078115.999839:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.999847:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078115.999891:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612775040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078115.999918:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.999922:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078115.999924:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.999928:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5104 00000100:00080000:0.0:1769078115.999931:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.999935:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078115.999945:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078115.999956:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612775168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078115.999964:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612775040:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078115.999984:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.999988:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078115.999990:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.999994:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5104 00000100:00080000:0.0:1769078115.999997:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.000001:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078116.000005:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078116.000015:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612775296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078116.000024:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078116.000027:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078116.000029:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078116.000032:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5104 00000100:00080000:0.0:1769078116.000042:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078116.000045:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078116.000049:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769078116.000057:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078116.000061:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612775424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078116.000105:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078116] req@ffff8b1006026300 x1855007612775040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078116.000120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612775040/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078116.000128:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078116.000131:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078116.000134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612775040:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078116.000156:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775168:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078116.000173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078116.000181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078116] req@ffff8b1006026a00 x1855007612775168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078116.000189:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612775168/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078116.000194:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078116.000197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078116.000198:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775168:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078116.000202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078116.000224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078116.000234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078116] req@ffff8b1006024000 x1855007612775296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078116.000253:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612775296/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078116.000261:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078116.000264:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078116.000268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078116.000274:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775424:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078116.000299:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078116.000307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078116] req@ffff8b1006025880 x1855007612775424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078116.000314:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612775424/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078116.000319:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078116.000321:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078116.000324:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775424:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078116.895780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078116.895829:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078117.919736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078117.919795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078118.943762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078118.943815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078119.967699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078119.967789:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078120.991758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078120.991818:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078121.119771:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.119794:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078121.119799:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.119811:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5109 00000100:00080000:0.0:1769078121.119825:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.119833:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078121.119878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612775552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.119910:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.119914:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.119915:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.119920:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5109 00000100:00080000:0.0:1769078121.119924:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.119928:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078121.119939:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.119954:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612775680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078121.119959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612775552:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078121.119969:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.119974:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.119976:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.119980:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5109 00000100:00080000:0.0:1769078121.119984:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.119989:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078121.119994:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.120007:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612775808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.120016:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.120020:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078121.120022:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.120025:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5109 00000100:00080000:0.0:1769078121.120029:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078121.120034:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078121.120037:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078121.120043:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.120060:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612775936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078121.120083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff8b1006026680 x1855007612775552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078121.120100:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612775552/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078121.120109:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078121.120113:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078121.120116:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612775552:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078121.120137:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775680:192.168.204.102@tcp:38:kworker.0 00000800:00000100:3.0:1769078121.120159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078121.120167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff8b1006025180 x1855007612775680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078121.120175:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612775680/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078121.120181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078121.120183:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078121.120185:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775680:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078121.120189:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078121.120206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078121.120213:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff8b1006027800 x1855007612775808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078121.120220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612775808/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078121.120229:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078121.120232:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078121.120233:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078121.120237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775936:192.168.204.102@tcp:8:kworker.0 00000800:00000100:3.0:1769078121.120247:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078121.120252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff8b1006027100 x1855007612775936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078121.120259:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612775936/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078121.120264:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078121.120266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:3.0:1769078121.120267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612775936:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078122.015736:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078122.015810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078123.039756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078123.039825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078124.063725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078124.063832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078125.087748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078125.087811:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078126.111753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078126.111832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078126.239792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078126.239805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078126.239810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078126.239820:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5114 00000100:00080000:3.0:1769078126.239833:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078126.239840:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078126.239884:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612776064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078126.239918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078126.239922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078126.239923:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078126.239926:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5114 00000100:00080000:3.0:1769078126.239929:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078126.239932:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078126.239940:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078126.239950:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612776192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078126.239957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078126.239961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078126.239962:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078126.239965:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5114 00000100:00080000:3.0:1769078126.239969:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078126.239972:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078126.239977:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078126.239983:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612776320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078126.239991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078126.239994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078126.239996:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078126.240000:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5114 00000100:00080000:3.0:1769078126.240003:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078126.240008:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078126.240015:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078126.240016:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612776064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078126.240032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612776448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078126.240104:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078126.240148:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff8b10037dc000 x1855007612776064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078126.240168:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612776064/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078126.240181:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078126.240186:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078126.240191:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612776064:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078126.240201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776192:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078126.240228:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078126.240243:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff8b10037dd880 x1855007612776192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078126.240258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612776192/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078126.240267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078126.240271:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078126.240275:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776192:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078126.240282:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078126.240305:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078126.240317:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff8b10037df100 x1855007612776320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078126.240331:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612776320/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078126.240346:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078126.240350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078126.240354:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078126.240381:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776448:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078126.240404:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078126.240416:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff8b10037ddc00 x1855007612776448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078126.240430:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612776448/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078126.240439:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078126.240443:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078126.240446:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776448:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078127.135774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078127.135833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078128.159737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078128.159837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078129.183777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078129.183894:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078130.207747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078130.207829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078131.231814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078131.231861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078131.359730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078131.359738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078131.359742:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078131.359755:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5119 00000100:00080000:3.0:1769078131.359767:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078131.359782:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078131.359814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612776576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078131.359834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078131.359837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078131.359838:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078131.359841:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5119 00000100:00080000:3.0:1769078131.359844:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078131.359846:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078131.359854:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078131.359863:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078131.359871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078131.359874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078131.359877:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078131.359878:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612776576:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078131.359882:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5119 00000100:00080000:3.0:1769078131.359888:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078131.359891:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078131.359894:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078131.359902:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612776832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078131.359909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078131.359911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078131.359912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078131.359924:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5119 00000100:00080000:3.0:1769078131.359929:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078131.359931:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078131.359935:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078131.359942:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078131.359945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612776960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078131.359976:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff8b10037df800 x1855007612776576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078131.359990:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612776576/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.359998:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078131.360001:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078131.360004:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612776576:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078131.360023:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776704:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078131.360038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078131.360046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff8b10037dc700 x1855007612776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078131.360055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612776704/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.360064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078131.360068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078131.360071:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776704:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078131.360078:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078131.360100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078131.360112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff8b10037dce00 x1855007612776832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078131.360126:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612776832/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.360135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078131.360138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078131.360140:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078131.360151:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776960:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078131.360167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078131.360176:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff8b10037df480 x1855007612776960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078131.360184:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612776960/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.360190:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078131.360192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078131.360194:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612776960:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078132.255707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078132.255804:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078133.279780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078133.279874:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078134.303783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078134.303835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078135.327800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078135.327874:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078136.351782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078136.351876:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078136.479817:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078136.479829:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078136.479833:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078136.479844:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5124 00000100:00080000:0.0:1769078136.479856:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078136.479863:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078136.479905:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612777088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078136.479942:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078136.479946:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078136.479948:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078136.479953:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5124 00000100:00080000:0.0:1769078136.479957:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078136.479962:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078136.479972:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078136.479987:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612777088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078136.479990:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612777216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078136.480005:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078136.480008:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078136.480010:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078136.480014:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5124 00000100:00080000:0.0:1769078136.480018:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078136.480023:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078136.480030:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078136.480043:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612777344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078136.480046:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078136.480058:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078136.480063:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078136.480064:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078136.480069:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5124 00000100:00080000:0.0:1769078136.480074:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078136.480079:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078136.480086:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078136.480090:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff8b1006027100 x1855007612777088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.480101:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612777472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078136.480107:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612777088/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078136.480123:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078136.480127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078136.480130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612777088:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078136.480150:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777216:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078136.480173:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078136.480182:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff8b1006027800 x1855007612777216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078136.480191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612777216/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078136.480201:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078136.480203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078136.480206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777216:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078136.480210:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777344:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078136.480222:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078136.480229:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff8b1006025180 x1855007612777344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078136.480236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612777344/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078136.480242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078136.480243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078136.480245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078136.480249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777472:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078136.480266:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078136.480273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff8b1006026680 x1855007612777472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078136.480299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612777472/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078136.480304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078136.480306:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078136.480309:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777472:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078137.375794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078137.375895:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078138.399767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078138.399831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078139.423770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078139.423816:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078140.447785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078140.447861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078141.471759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078141.471788:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078141.599773:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078141.599785:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078141.599790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078141.599800:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5129 00000100:00080000:0.0:1769078141.599813:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078141.599820:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078141.599878:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612777600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078141.599913:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078141.599918:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078141.599921:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078141.599927:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5129 00000100:00080000:0.0:1769078141.599931:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078141.599936:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078141.599948:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078141.599967:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612777728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078141.599981:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078141.599987:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078141.599992:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078141.599994:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612777600:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078141.600000:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5129 00000100:00080000:0.0:1769078141.600013:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078141.600017:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078141.600023:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078141.600041:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612777856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078141.600049:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078141.600052:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078141.600053:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078141.600057:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5129 00000100:00080000:0.0:1769078141.600059:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078141.600062:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078141.600066:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078141.600076:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078141.600078:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612777984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078141.600124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff8b1006025880 x1855007612777600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078141.600147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612777600/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078141.600158:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078141.600163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078141.600168:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612777600:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078141.600224:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777728:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078141.600248:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078141.600258:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff8b1006024000 x1855007612777728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078141.600267:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612777728/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078141.600273:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078141.600276:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078141.600278:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777728:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078141.600283:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078141.600305:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078141.600317:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff8b1006026a00 x1855007612777856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078141.600339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612777856/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078141.600353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078141.600356:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078141.600359:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078141.600366:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777984:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078141.600388:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078141.600399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff8b1006026300 x1855007612777984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078141.600408:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612777984/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078141.600414:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078141.600416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078141.600418:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612777984:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078142.495798:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078142.495851:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078143.519763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078143.519838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078144.543797:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078144.543871:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078145.567735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078145.567808:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078146.591748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078146.591809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078146.719741:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078146.719748:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078146.719752:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078146.719760:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5134 00000100:00080000:2.0:1769078146.719770:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078146.719777:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078146.719808:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612778112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078146.719828:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078146.719831:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078146.719833:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078146.719837:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5134 00000100:00080000:2.0:1769078146.719840:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078146.719843:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078146.719851:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078146.719855:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612778112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078146.719866:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612778240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078146.719875:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078146.719878:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078146.719880:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078146.719883:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5134 00000100:00080000:2.0:1769078146.719887:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078146.719889:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078146.719893:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078146.719902:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612778368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078146.719909:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078146.719912:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078146.719913:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078146.719919:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5134 00000800:00000100:1.0:1769078146.719921:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769078146.719923:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078146.719927:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078146.719930:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078146.719938:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612778496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078146.719958:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff8b1005d58000 x1855007612778112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078146.719975:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612778112/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078146.719987:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078146.719990:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078146.719993:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612778112:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078146.720012:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778240:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078146.720029:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078146.720040:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff8b1005d59180 x1855007612778240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078146.720069:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612778240/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078146.720078:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078146.720082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078146.720085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778240:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078146.720092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078146.720126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078146.720137:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff8b1005d5a300 x1855007612778368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078146.720145:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612778368/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078146.720151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078146.720153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078146.720156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078146.720160:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778496:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078146.720174:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078146.720180:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff8b1005d58e00 x1855007612778496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078146.720188:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612778496/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078146.720194:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078146.720197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078146.720199:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778496:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078147.615749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078147.615859:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078148.639802:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078148.639867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078149.663747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078149.663824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078150.687782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078150.687849:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078151.711751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078151.711814:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078151.839811:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078151.839820:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078151.839824:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078151.839832:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5139 00000100:00080000:1.0:1769078151.839842:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078151.839847:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078151.839880:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612778624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078151.839905:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078151.839908:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078151.839909:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078151.839912:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5139 00000100:00080000:1.0:1769078151.839915:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078151.839919:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078151.839926:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078151.839936:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612778752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078151.839943:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612778624:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078151.839946:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078151.839952:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078151.839953:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078151.839956:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5139 00000100:00080000:1.0:1769078151.839958:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078151.839960:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078151.839964:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078151.839973:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612778880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078151.839979:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078151.839981:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078151.839982:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078151.839985:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5139 00000100:00080000:1.0:1769078151.839987:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078151.839989:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078151.839993:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078151.839998:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612779008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078151.840017:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078151.840068:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff8b1006135c00 x1855007612778624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.840087:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612778624/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.840098:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.840102:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078151.840106:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612778624:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078151.840132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778752:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078151.840155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078151.840167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff8b1006137b80 x1855007612778752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.840179:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612778752/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.840186:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.840190:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078151.840193:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778752:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078151.840200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078151.840220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078151.840231:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff8b1006135880 x1855007612778880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.840243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612778880/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.840252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.840255:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078151.840257:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612778880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078151.840262:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779008:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078151.840277:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078151.840284:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff8b1006135500 x1855007612779008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.840292:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612779008/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.840298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.840300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078151.840302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779008:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078152.735756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078152.735882:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078153.759759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078153.759881:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078154.783753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078154.783819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078155.807790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078155.807837:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078156.831766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078156.831826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078156.959797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078156.959808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078156.959813:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078156.959831:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5145 00000100:00080000:0.0:1769078156.959845:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078156.959854:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078156.959901:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612779136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078156.959926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078156.959928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078156.959930:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078156.959943:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5145 00000100:00080000:0.0:1769078156.959948:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078156.959951:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078156.959962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078156.959976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612779264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078156.959988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078156.959990:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612779136:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078156.959994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078156.959998:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078156.960005:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5145 00000100:00080000:0.0:1769078156.960008:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078156.960011:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078156.960016:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078156.960025:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612779392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078156.960032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078156.960034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078156.960036:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078156.960039:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5145 00000100:00080000:0.0:1769078156.960043:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078156.960047:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078156.960051:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078156.960062:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612779520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078156.960071:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078156.960112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff8b1006026d80 x1855007612779136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.960128:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612779136/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.960136:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.960140:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078156.960143:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612779136:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078156.960169:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779264:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078156.960191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078156.960205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff8b1006027b80 x1855007612779264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.960215:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612779264/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.960221:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.960223:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078156.960225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779264:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078156.960229:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078156.960244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078156.960250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff8b1006025f80 x1855007612779392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.960262:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612779392/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.960267:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.960268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078156.960270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078156.960273:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779520:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078156.960284:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078156.960289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff8b1006024a80 x1855007612779520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.960296:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612779520/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.960301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.960303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078156.960304:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779520:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078157.855773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078157.855832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078158.879739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078158.879805:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078159.903769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078159.903844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078160.927730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078160.927787:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078161.951783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078161.951872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078162.079762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078162.079770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078162.079774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078162.079782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5150 00000100:00080000:0.0:1769078162.079792:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078162.079798:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078162.079828:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612779648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078162.079864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078162.079867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078162.079868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078162.079872:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5150 00000100:00080000:0.0:1769078162.079876:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078162.079879:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078162.079887:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078162.079897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612779776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078162.079906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078162.079909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078162.079911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078162.079915:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5150 00000100:00080000:0.0:1769078162.079922:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078162.079923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612779648:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078162.079926:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078162.079942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078162.079959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612779904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078162.079965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078162.079968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078162.079969:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078162.079971:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5150 00000100:00080000:0.0:1769078162.079974:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078162.079977:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078162.079980:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078162.079991:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612780032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078162.080058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078162.080085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078162.080102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078162.975726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078162.975785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078163.999775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078163.999864:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078165.023738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078165.023863:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769078165.152089:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769078165.152157:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769078165.152183:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078165.152236:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078165.152248:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078165.152260:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769078165.152301:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078165] req@ffff8b1006024700 x1855007612779776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078165.152352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612779776/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078165.152370:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078165.152377:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078165.152383:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779776:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078165.152396:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078165] req@ffff8b1006024e00 x1855007612779904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078165.152410:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612779904/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078165.152419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078165.152423:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078165.152427:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612779904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078165.152435:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078165] req@ffff8b1006024380 x1855007612780032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078165.152450:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612780032/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078165.152459:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078165.152463:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078165.152466:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780032:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078165.152475:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078165] req@ffff8b1006027480 x1855007612779648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078165.152492:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612779648/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078165.152501:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078165.152505:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078165.152509:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612779648:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078166.047738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078166.047819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078167.071784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078167.071859:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078167.199883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.199896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078167.199901:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.199914:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5155 00000100:00080000:1.0:1769078167.199930:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.199940:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078167.199991:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612780160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078167.200027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.200032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078167.200034:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.200038:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5155 00000100:00080000:1.0:1769078167.200043:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.200047:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078167.200060:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078167.200076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612780288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078167.200088:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612780160:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078167.200092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.200099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078167.200103:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.200108:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5155 00000100:00080000:1.0:1769078167.200113:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.200118:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078167.200125:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078167.200138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612780416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078167.200148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.200153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078167.200155:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.200159:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5155 00000100:00080000:1.0:1769078167.200164:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.200169:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078167.200174:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078167.200188:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612780544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078167.200198:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078167.200251:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff8b1006136d80 x1855007612780160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.200277:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612780160/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.200299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.200305:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078167.200313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612780160:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078167.200349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780288:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078167.200379:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078167.200395:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff8b1006136680 x1855007612780288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.200411:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612780288/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.200419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.200424:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078167.200428:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780288:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078167.200435:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078167.200459:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078167.200472:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff8b1006137100 x1855007612780416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.200487:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612780416/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.200497:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.200501:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078167.200504:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078167.200511:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780544:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078167.200545:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078167.200558:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff8b1006134e00 x1855007612780544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.200569:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612780544/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.200578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.200581:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078167.200585:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780544:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078168.095781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078168.095843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078169.119781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078169.119846:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078170.143777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078170.143833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078171.167765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078171.167851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078172.191771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078172.191855:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078172.319745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078172.319763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078172.319767:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078172.319775:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5160 00000100:00080000:3.0:1769078172.319785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078172.319790:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078172.319820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612780672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078172.319842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078172.319845:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078172.319847:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078172.319852:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5160 00000100:00080000:3.0:1769078172.319856:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078172.319860:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078172.319870:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078172.319884:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612780672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078172.319889:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612780800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078172.319905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078172.319908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078172.319911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078172.319916:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5160 00000100:00080000:3.0:1769078172.319921:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078172.319925:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078172.319931:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078172.319945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612780928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078172.319952:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078172.319963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078172.319968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078172.319970:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078172.319976:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5160 00000100:00080000:3.0:1769078172.319981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078172.319986:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078172.319989:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff8b10037dea00 x1855007612780672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078172.319994:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078172.320009:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612780672/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078172.320015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612781056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078172.320019:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.320029:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078172.320036:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612780672:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078172.320061:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780800:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078172.320087:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078172.320101:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff8b10037df480 x1855007612780800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.320116:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612780800/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.320125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.320129:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078172.320132:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780800:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078172.320145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078172.320165:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078172.320175:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff8b10037dce00 x1855007612780928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.320194:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612780928/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.320202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.320205:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078172.320209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612780928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078172.320215:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781056:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078172.320236:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078172.320247:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff8b10037dc700 x1855007612781056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.320260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612781056/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.320269:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.320273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078172.320277:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781056:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078173.215766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078173.215835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078174.239739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078174.239802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078175.263758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078175.263832:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078176.287767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078176.287829:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078177.311839:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078177.312079:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078177.439765:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078177.439777:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078177.439781:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078177.439792:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5165 00000100:00080000:2.0:1769078177.439805:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078177.439812:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078177.439858:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612781184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078177.439885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078177.439889:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078177.439890:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078177.439893:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5165 00000100:00080000:2.0:1769078177.439896:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078177.439898:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078177.439908:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078177.439917:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612781312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078177.439925:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078177.439929:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078177.439930:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078177.439934:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5165 00000100:00080000:2.0:1769078177.439936:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078177.439939:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078177.439943:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078177.439950:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612781440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078177.439959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612781184:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078177.439961:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078177.439969:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078177.439972:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078177.439976:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5165 00000100:00080000:2.0:1769078177.439978:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078177.439981:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078177.439985:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078177.439994:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612781568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078177.440048:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078177.440089:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078177] req@ffff8b1005d58a80 x1855007612781184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078177.440109:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612781184/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078177.440121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078177.440126:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078177.440130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612781184:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078177.440139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781312:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078177.440162:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078177.440177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078177] req@ffff8b1005d5b800 x1855007612781312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078177.440191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612781312/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078177.440199:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078177.440203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078177.440207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781312:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078177.440250:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078177.440275:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078177.440289:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078177] req@ffff8b1005d59500 x1855007612781440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078177.440303:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612781440/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078177.440311:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078177.440315:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078177.440331:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078177.440338:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781568:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078177.440361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078177.440373:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078177] req@ffff8b1005d5b480 x1855007612781568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078177.440399:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612781568/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078177.440413:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078177.440416:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078177.440421:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781568:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078178.335734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078178.335809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078179.359787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078179.359871:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078180.383759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078180.383824:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078181.407770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078181.407837:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078182.431732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078182.431793:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078182.559791:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078182.559802:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078182.559807:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078182.559819:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5170 00000100:00080000:2.0:1769078182.559832:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078182.559839:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078182.559881:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612781696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078182.559910:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078182.559914:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078182.559916:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078182.559921:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5170 00000100:00080000:2.0:1769078182.559924:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078182.559928:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078182.559938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078182.559953:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612781824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078182.559960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078182.559965:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078182.559967:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078182.559974:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5170 00000100:00080000:2.0:1769078182.559980:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078182.559987:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078182.559990:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612781696:192.168.204.102@tcp:250:kworker.0 00000008:00100000:2.0:1769078182.559992:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078182.560008:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612781952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078182.560020:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078182.560023:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078182.560025:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078182.560031:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5170 00000100:00080000:2.0:1769078182.560037:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078182.560041:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078182.560046:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078182.560054:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612782080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078182.560082:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078182.560124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff8b1005d5bb80 x1855007612781696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078182.560151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612781696/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078182.560159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078182.560163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078182.560165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612781696:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078182.560171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781824:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078182.560196:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078182.560206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff8b1005d59880 x1855007612781824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078182.560213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612781824/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078182.560223:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078182.560225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078182.560228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781824:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078182.560248:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078182.560262:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078182.560273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff8b1005d5aa00 x1855007612781952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078182.560285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612781952/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078182.560293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078182.560297:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078182.560301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612781952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078182.560307:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782080:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078182.560328:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078182.560338:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff8b1005d5b100 x1855007612782080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078182.560347:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612782080/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078182.560352:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078182.560354:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078182.560357:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782080:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078183.455755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078183.455815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078184.479763:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078184.479833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078185.503773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078185.503947:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078186.527746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078186.527782:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078187.551775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078187.551833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078187.679802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078187.679817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078187.679823:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078187.679836:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5175 00000100:00080000:3.0:1769078187.679850:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078187.679859:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078187.679903:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612782208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078187.679930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078187.679933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078187.679934:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078187.679937:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5175 00000100:00080000:3.0:1769078187.679940:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078187.679943:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078187.679952:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078187.679960:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612782336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078187.679968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078187.679971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078187.679973:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078187.679978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5175 00000100:00080000:3.0:1769078187.679982:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078187.679986:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078187.679992:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078187.680004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612782464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078187.680015:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612782208:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078187.680029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078187.680034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078187.680035:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078187.680039:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5175 00000100:00080000:3.0:1769078187.680042:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078187.680045:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078187.680049:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078187.680061:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612782592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078187.680113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078187.680150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff8b10037df800 x1855007612782208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078187.680167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612782208/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078187.680177:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078187.680182:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078187.680186:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612782208:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078187.680194:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782336:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078187.680214:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078187.680224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff8b10037ddc00 x1855007612782336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078187.680232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612782336/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078187.680237:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078187.680239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078187.680241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782336:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078187.680261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078187.680274:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078187.680280:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff8b10037df100 x1855007612782464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078187.680291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612782464/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078187.680298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078187.680302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078187.680305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078187.680311:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782592:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078187.680330:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078187.680340:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff8b10037dd880 x1855007612782592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078187.680352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612782592/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078187.680360:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078187.680363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078187.680367:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782592:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078188.575733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078188.575782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078189.599796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078189.599812:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078190.623796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078190.623846:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078191.647787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078191.647857:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078192.671810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078192.671811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:0.0:1769078192.799768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078192.799782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078192.799787:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078192.799799:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5180 00000100:00080000:0.0:1769078192.799814:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078192.799822:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078192.799882:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612782720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078192.799920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078192.799925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078192.799927:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078192.799932:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5180 00000100:00080000:0.0:1769078192.799936:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078192.799940:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078192.799953:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078192.799967:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612782848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078192.799979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078192.799982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078192.799984:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078192.799991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5180 00000100:00080000:0.0:1769078192.799997:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078192.799997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612782720:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078192.800014:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078192.800022:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078192.800038:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612782976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078192.800048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078192.800052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078192.800054:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078192.800058:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5180 00000100:00080000:0.0:1769078192.800071:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078192.800078:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769078192.800079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769078192.800090:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078192.800106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612783104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078192.800131:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff8b1006027480 x1855007612782720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078192.800151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612782720/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078192.800161:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078192.800166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078192.800170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612782720:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078192.800195:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782848:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078192.800218:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078192.800231:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff8b1006024380 x1855007612782848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078192.800250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612782848/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078192.800258:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078192.800261:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078192.800265:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782848:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078192.800271:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078192.800291:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078192.800301:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff8b1006024e00 x1855007612782976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078192.800313:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612782976/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078192.800320:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078192.800324:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078192.800327:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612782976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078192.800359:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783104:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078192.800379:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078192.800390:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff8b1006024700 x1855007612783104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078192.800402:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612783104/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078192.800416:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078192.800419:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078192.800422:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783104:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078193.695725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078193.695791:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078194.719785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078194.719830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078195.743769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078195.743801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078196.767771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078196.767814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078197.791728:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078197.791783:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078197.919762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.919770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078197.919775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.919782:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5186 00000100:00080000:1.0:1769078197.919791:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.919796:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078197.919832:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612783232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.919853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.919856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078197.919857:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.919860:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5186 00000100:00080000:1.0:1769078197.919862:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.919864:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078197.919869:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078197.919881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612783360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.919889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078197.919892:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612783232:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078197.919892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078197.919896:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.919901:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5186 00000100:00080000:1.0:1769078197.919903:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.919905:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078197.919908:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078197.919917:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612783488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.919923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.919925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078197.919926:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.919928:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5186 00000100:00080000:1.0:1769078197.919945:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.919947:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078197.919949:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078197.919953:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078197.919958:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612783616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078197.919986:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff8b1006134a80 x1855007612783232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078197.919999:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612783232/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078197.920008:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078197.920012:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078197.920016:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612783232:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078197.920036:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783360:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078197.920057:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078197.920069:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff8b1006137480 x1855007612783360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078197.920081:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612783360/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078197.920089:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078197.920092:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078197.920096:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783360:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078197.920102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078197.920120:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078197.920129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff8b1006134700 x1855007612783488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078197.920140:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612783488/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078197.920148:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078197.920151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078197.920154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078197.920160:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783616:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078197.920178:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078197.920189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff8b1006134e00 x1855007612783616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078197.920200:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612783616/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078197.920207:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078197.920211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078197.920214:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783616:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078198.815738:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078198.815801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078199.839720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078199.839763:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078200.863766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078200.863833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078201.887782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078201.887843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078202.911783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078202.911822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078203.039731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078203.039750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078203.039754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078203.039764:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5191 00000100:00080000:1.0:1769078203.039775:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078203.039782:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078203.039814:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612783744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078203.039840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078203.039842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078203.039844:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078203.039846:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5191 00000100:00080000:1.0:1769078203.039849:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078203.039851:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078203.039858:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078203.039867:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612783872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078203.039873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078203.039876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078203.039877:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078203.039879:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5191 00000100:00080000:1.0:1769078203.039881:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078203.039883:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078203.039886:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078203.039894:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612784000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078203.039896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612783744:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078203.039903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078203.039906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078203.039907:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078203.039909:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5191 00000100:00080000:1.0:1769078203.039911:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078203.039913:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078203.039916:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078203.039924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612784128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078203.039965:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078203.039996:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff8b1006137100 x1855007612783744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078203.040010:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612783744/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078203.040017:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078203.040020:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078203.040022:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612783744:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078203.040027:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783872:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078203.040042:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078203.040050:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff8b1006136680 x1855007612783872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078203.040062:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612783872/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078203.040066:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078203.040068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078203.040070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612783872:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078203.040087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784000:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078203.040097:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078203.040103:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff8b1006136d80 x1855007612784000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078203.040110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612784000/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078203.040114:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078203.040116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078203.040118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078203.040121:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784128:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078203.040129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078203.040135:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078203/real 1769078203] req@ffff8b1006135500 x1855007612784128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078203.040141:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612784128/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078203.040150:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078203.040152:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078203.040154:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784128:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078203.935775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078203.935856:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078204.959779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078204.959822:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078205.983779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078205.983846:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078207.007731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078207.007792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078208.031709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078208.031753:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078208.159732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.159740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078208.159744:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.159751:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5196 00000100:00080000:0.0:1769078208.159760:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.159766:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078208.159795:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612784256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078208.159817:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.159819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078208.159820:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.159823:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5196 00000100:00080000:0.0:1769078208.159826:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.159829:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078208.159836:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078208.159846:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612784384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078208.159847:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612784256:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078208.159860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.159863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078208.159864:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.159867:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5196 00000100:00080000:0.0:1769078208.159869:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.159871:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078208.159874:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078208.159883:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612784512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078208.159888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.159890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078208.159891:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.159893:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5196 00000100:00080000:0.0:1769078208.159896:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.159898:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078208.159900:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078208.159904:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078208.159908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612784640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078208.159937:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff8b1006024a80 x1855007612784256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078208.159949:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612784256/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078208.159956:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078208.159959:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078208.159962:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612784256:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078208.159977:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784384:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078208.159990:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078208.159997:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff8b1006025f80 x1855007612784384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078208.160003:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612784384/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078208.160008:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078208.160010:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078208.160013:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784384:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078208.160016:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078208.160026:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078208.160031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff8b1006027b80 x1855007612784512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078208.160038:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612784512/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078208.160043:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078208.160046:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078208.160049:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078208.160054:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784640:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078208.160082:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078208.160094:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff8b1006026d80 x1855007612784640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078208.160104:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612784640/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078208.160111:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078208.160114:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078208.160117:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784640:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078209.055766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078209.055826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078210.079724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078210.079781:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078211.103773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078211.103835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078212.127700:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078212.127752:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078213.151735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078213.151791:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078213.279779:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078213.279790:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078213.279795:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078213.279804:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5201 00000100:00080000:2.0:1769078213.279814:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078213.279821:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078213.279858:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612784768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078213.279893:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078213.279901:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078213.279903:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078213.279907:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5201 00000100:00080000:2.0:1769078213.279911:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078213.279915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612784768:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078213.279919:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078213.279931:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078213.279945:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612784896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078213.279956:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078213.279959:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078213.279961:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078213.279964:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5201 00000100:00080000:2.0:1769078213.279969:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078213.279973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078213.279977:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078213.279980:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078213.280009:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612785024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078213.280019:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078213.280022:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078213.280024:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078213.280028:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5201 00000100:00080000:2.0:1769078213.280033:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078213.280037:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078213.280042:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078213.280043:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff8b1005d5ad80 x1855007612784768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.280056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612785152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078213.280066:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612784768/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078213.280077:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078213.280082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078213.280087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612784768:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078213.280108:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784896:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078213.280128:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078213.280139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff8b1005d58380 x1855007612784896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078213.280149:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612784896/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078213.280156:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078213.280159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078213.280162:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612784896:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078213.280167:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078213.280183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078213.280192:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff8b1005d5b100 x1855007612785024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078213.280202:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612785024/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078213.280214:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078213.280217:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078213.280220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078213.280225:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785152:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078213.280240:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078213.280249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff8b1005d5aa00 x1855007612785152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078213.280258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612785152/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078213.280266:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078213.280268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078213.280271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785152:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078214.175743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078214.175802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078215.199739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078215.199809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078216.223796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078216.223869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078217.247715:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078217.247758:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078218.271730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078218.271792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078218.399727:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078218.399736:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078218.399741:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078218.399750:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5206 00000100:00080000:2.0:1769078218.399760:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078218.399767:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078218.399797:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612785280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078218.399830:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078218.399833:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078218.399835:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078218.399839:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5206 00000100:00080000:2.0:1769078218.399843:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078218.399847:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078218.399855:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078218.399867:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612785280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078218.399869:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612785408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078218.399890:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078218.399894:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078218.399897:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078218.399916:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5206 00000100:00080000:2.0:1769078218.399920:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078218.399924:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078218.399929:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078218.399940:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612785536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078218.399950:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769078218.399950:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078218.399956:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078218.399958:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078218.399962:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5206 00000100:00080000:2.0:1769078218.399967:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078218.399973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078218.399977:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078218.399988:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612785664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078218.399990:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff8b1005d59880 x1855007612785280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078218.400022:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612785280/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078218.400032:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078218.400038:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078218.400042:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612785280:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078218.400065:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785408:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078218.400086:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078218.400098:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff8b1005d5bb80 x1855007612785408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078218.400106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612785408/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078218.400113:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078218.400115:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078218.400118:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785408:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078218.400122:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078218.400135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078218.400143:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff8b1005d5b480 x1855007612785536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078218.400151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612785536/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078218.400169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078218.400172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078218.400175:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078218.400181:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785664:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078218.400200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078218.400210:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff8b1005d59500 x1855007612785664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078218.400221:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612785664/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078218.400226:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078218.400228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078218.400229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785664:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078219.295734:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078219.295770:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078220.319735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078220.319779:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078221.343709:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078221.343779:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078222.367741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078222.367801:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078223.391724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078223.391771:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078223.519794:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078223.519803:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078223.519807:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078223.519835:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5211 00000100:00080000:3.0:1769078223.519845:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078223.519851:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078223.519888:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612785792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078223.519918:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078223.519922:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078223.519924:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078223.519928:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5211 00000100:00080000:3.0:1769078223.519931:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078223.519935:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078223.519952:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078223.519959:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612785792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078223.519966:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612785920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078223.519978:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078223.519981:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078223.519983:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078223.519987:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5211 00000100:00080000:3.0:1769078223.519990:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078223.519994:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078223.519999:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078223.520010:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612786048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078223.520013:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078223.520025:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078223.520028:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078223.520030:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078223.520036:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5211 00000100:00080000:3.0:1769078223.520040:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078223.520044:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff8b10037dc000 x1855007612785792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078223.520048:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078223.520057:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078223.520064:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612785792/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078223.520077:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078223.520079:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612786176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078223.520082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078223.520089:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612785792:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078223.520111:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785920:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078223.520128:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078223.520139:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff8b10037dd500 x1855007612785920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078223.520151:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612785920/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078223.520159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078223.520162:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078223.520165:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612785920:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078223.520172:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078223.520189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078223.520196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff8b10037ddf80 x1855007612786048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078223.520203:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612786048/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078223.520208:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078223.520210:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078223.520212:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078223.520216:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786176:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078223.520225:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078223.520231:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff8b10037de300 x1855007612786176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078223.520237:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612786176/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078223.520242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078223.520243:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078223.520245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786176:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078224.415764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078224.415852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078225.439770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078225.439848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078226.463741:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078226.463802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078227.487765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078227.487821:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078228.511811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078228.511872:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078228.639764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078228.639775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078228.639780:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078228.639789:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5216 00000100:00080000:1.0:1769078228.639799:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078228.639806:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078228.639841:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612786304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078228.639866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078228.639870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078228.639872:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078228.639876:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5216 00000100:00080000:1.0:1769078228.639880:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078228.639885:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078228.639894:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078228.639902:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612786304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078228.639907:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612786432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078228.639919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078228.639923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078228.639925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078228.639929:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5216 00000100:00080000:1.0:1769078228.639933:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078228.639936:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078228.639941:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078228.639951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612786560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078228.639960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078228.639963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078228.639964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078228.639968:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5216 00000100:00080000:1.0:1769078228.639972:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078228.639976:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078228.639981:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078228.639991:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612786688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078228.639997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078228.640035:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078228.640073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078229.535793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078229.535853:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078230.559770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078230.559830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078231.583807:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078231.583812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769078231.711942:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769078231.711981:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769078231.711997:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078231.712033:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078231.712040:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078231.712046:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769078231.712088:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078231] req@ffff8b1006136a00 x1855007612786688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078231.712112:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612786688/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078231.712125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078231.712130:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078231.712134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786688:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078231.712140:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078231] req@ffff8b1006137b80 x1855007612786432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078231.712152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612786432/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078231.712159:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078231.712163:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078231.712166:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786432:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078231.712172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078231] req@ffff8b1006135c00 x1855007612786560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078231.712183:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612786560/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078231.712191:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078231.712194:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078231.712197:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078231.712202:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078231] req@ffff8b1006135880 x1855007612786304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078231.712213:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612786304/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078231.712220:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078231.712225:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078231.712228:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612786304:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078232.607779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078232.607861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078233.631783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078233.631844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078233.759802:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078233.759815:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078233.759820:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078233.759833:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5221 00000100:00080000:3.0:1769078233.759848:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078233.759856:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078233.759900:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612786816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078233.759944:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078233.759948:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078233.759950:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078233.759955:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5221 00000100:00080000:3.0:1769078233.759959:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078233.759963:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078233.759975:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078233.759989:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612786944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078233.759999:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078233.760003:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078233.760005:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078233.760009:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5221 00000100:00080000:3.0:1769078233.760013:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078233.760017:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078233.760023:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078233.760037:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612787072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078233.760046:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078233.760050:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078233.760051:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078233.760055:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5221 00000100:00080000:3.0:1769078233.760059:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078233.760063:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078233.760068:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078233.760079:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612787200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078233.760200:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612786816:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769078233.760289:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078233.760346:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff8b10037ded80 x1855007612786816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.760366:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612786816/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.760377:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.760381:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078233.760385:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612786816:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078233.760393:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786944:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078233.760417:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078233.760431:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff8b10037dca80 x1855007612786944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.760444:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612786944/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.760452:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.760455:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078233.760459:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612786944:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078233.760465:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078233.760486:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078233.760497:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff8b10037dc380 x1855007612787072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.760508:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612787072/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.760516:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.760519:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078233.760522:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078233.760528:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787200:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078233.760548:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078233.760560:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff8b10037dd180 x1855007612787200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.760570:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612787200/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.760578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.760581:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078233.760584:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787200:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078234.655735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078234.655794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078235.679708:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078235.679760:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078236.703699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078236.703758:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078237.727731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078237.727800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078238.751789:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078238.751825:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078238.879750:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.879758:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078238.879762:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.879769:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5226 00000100:00080000:3.0:1769078238.879778:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.879784:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078238.879814:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612787328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078238.879835:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.879838:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078238.879839:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.879842:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5226 00000100:00080000:3.0:1769078238.879844:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.879846:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078238.879852:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078238.879862:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612787456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078238.879868:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.879871:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078238.879872:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.879875:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5226 00000100:00080000:3.0:1769078238.879876:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.879878:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078238.879881:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078238.879888:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612787328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078238.879889:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612787584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078238.879904:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.879906:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078238.879907:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.879909:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5226 00000100:00080000:3.0:1769078238.879911:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.879913:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078238.879916:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078238.879922:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612787712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078238.879950:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078238.879976:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff8b10037de680 x1855007612787328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.879989:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612787328/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.879995:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.879998:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078238.880002:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612787328:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078238.880009:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787456:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078238.880036:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078238.880047:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff8b10037dd180 x1855007612787456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.880059:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612787456/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.880064:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.880066:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078238.880069:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787456:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078238.880092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078238.880113:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078238.880124:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff8b10037dc380 x1855007612787584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.880132:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612787584/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.880139:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.880142:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078238.880145:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787584:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078238.880150:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787712:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078238.880168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078238.880179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff8b10037dca80 x1855007612787712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.880197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612787712/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.880202:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.880204:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078238.880206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787712:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078239.775743:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078239.775803:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078240.799750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078240.799820:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078241.823759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078241.823890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078242.847724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078242.847782:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078243.871759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078243.871849:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078243.999773:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.999785:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078243.999790:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.999802:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5232 00000100:00080000:3.0:1769078243.999815:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.999828:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078243.999871:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612787840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078243.999900:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.999903:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078243.999905:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.999909:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5232 00000100:00080000:3.0:1769078243.999912:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.999916:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078243.999927:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078243.999938:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612787968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078243.999945:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612787840:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078243.999960:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.999964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078243.999966:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.999971:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5232 00000100:00080000:3.0:1769078243.999974:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.999978:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078243.999983:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078243.999993:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612788096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078244.000003:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078244.000006:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078244.000007:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078244.000010:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5232 00000100:00080000:3.0:1769078244.000014:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078244.000017:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078244.000021:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078244.000030:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612788224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078244.000033:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078244.000072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078244] req@ffff8b10037ded80 x1855007612787840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078244.000089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612787840/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078244.000100:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078244.000104:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078244.000108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612787840:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078244.000144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787968:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078244.000168:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078244.000181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078244] req@ffff8b10037de300 x1855007612787968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078244.000195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612787968/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078244.000203:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078244.000206:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078244.000208:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612787968:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078244.000213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078244.000229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078244.000236:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078244] req@ffff8b10037ddf80 x1855007612788096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078244.000247:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612788096/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078244.000253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078244.000254:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078244.000256:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078244.000259:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788224:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078244.000269:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078244.000274:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078244] req@ffff8b10037dd500 x1855007612788224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078244.000280:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612788224/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078244.000285:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078244.000286:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078244.000288:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788224:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078244.895745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078244.895821:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078245.919833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078245.919839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078246.943770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078246.943815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078247.967761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078247.967839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078248.991723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078248.991787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078249.119815:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.119828:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078249.119833:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.119853:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5237 00000100:00080000:1.0:1769078249.119868:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.119876:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078249.119926:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135180 x1855007612788352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.119959:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.119963:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078249.119965:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.119969:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5237 00000100:00080000:1.0:1769078249.119972:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.119976:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078249.119987:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078249.120008:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137800 x1855007612788480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.120019:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.120026:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769078249.120030:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612788352:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078249.120031:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.120044:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5237 00000100:00080000:1.0:1769078249.120050:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.120054:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078249.120059:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078249.120076:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136300 x1855007612788608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.120086:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.120090:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078249.120092:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.120097:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5237 00000100:00080000:1.0:1769078249.120111:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.120120:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:0.0:1769078249.120123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:1.0:1769078249.120130:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078249.120146:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135880 x1855007612788736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078249.120178:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff8b1006135180 x1855007612788352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078249.120199:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135180 x1855007612788352/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078249.120210:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078249.120221:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078249.120225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612788352:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078249.120248:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788480:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078249.120269:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078249.120281:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff8b1006137800 x1855007612788480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078249.120293:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137800 x1855007612788480/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078249.120301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078249.120304:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078249.120307:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788480:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078249.120314:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078249.120341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078249.120351:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff8b1006136300 x1855007612788608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078249.120362:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136300 x1855007612788608/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078249.120376:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078249.120379:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078249.120395:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078249.120402:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788736:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078249.120420:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078249.120429:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff8b1006135880 x1855007612788736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078249.120441:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135880 x1855007612788736/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078249.120448:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078249.120452:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078249.120455:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788736:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078250.015792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078250.015854:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078251.039770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078251.039794:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078252.063753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078252.063800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078253.087815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078253.087842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078254.111843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078254.111905:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078254.239777:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078254.239787:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078254.239793:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078254.239804:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5242 00000100:00080000:3.0:1769078254.239815:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078254.239824:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078254.239869:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612788864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078254.239912:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078254.239917:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078254.239919:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078254.239925:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5242 00000100:00080000:3.0:1769078254.239930:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078254.239935:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078254.239947:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078254.239953:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612788864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078254.239964:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612788992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078254.239977:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078254.239981:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078254.239990:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078254.239994:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5242 00000100:00080000:3.0:1769078254.239999:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078254.240003:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078254.240009:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078254.240022:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612789120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078254.240033:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078254.240041:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078254.240045:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078254.240046:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078254.240051:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5242 00000100:00080000:3.0:1769078254.240055:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078254.240059:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078254.240063:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078254.240078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff8b10037dc000 x1855007612788864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078254.240084:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612789248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078254.240110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612788864/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078254.240121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078254.240127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078254.240131:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612788864:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078254.240158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788992:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078254.240180:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078254.240196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff8b10037dd880 x1855007612788992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078254.240209:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612788992/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078254.240224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078254.240227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078254.240231:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612788992:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078254.240237:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078254.240260:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078254.240272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff8b10037df100 x1855007612789120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078254.240284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612789120/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078254.240292:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078254.240307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078254.240311:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078254.240319:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789248:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078254.240341:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078254.240355:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff8b10037ddc00 x1855007612789248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078254.240368:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612789248/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078254.240376:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078254.240381:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078254.240384:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789248:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078255.135756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078255.135818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078256.159742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078256.159807:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078257.183760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078257.183820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078258.207699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078258.207793:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078259.231754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078259.231810:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078259.359762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078259.359784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078259.359790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078259.359801:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5247 00000100:00080000:0.0:1769078259.359813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078259.359821:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078259.359877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612789376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078259.359907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078259.359910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078259.359914:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078259.359917:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5247 00000100:00080000:0.0:1769078259.359920:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078259.359923:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078259.359932:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078259.359942:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612789504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078259.359949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078259.359952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078259.359953:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078259.359955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5247 00000100:00080000:0.0:1769078259.359957:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078259.359961:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078259.359962:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612789376:192.168.204.102@tcp:250:kworker.0 00000008:00100000:0.0:1769078259.359964:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078259.359984:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612789632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078259.360000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078259.360004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078259.360006:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078259.360010:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5247 00000100:00080000:0.0:1769078259.360035:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078259.360039:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078259.360043:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078259.360048:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078259.360054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612789760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078259.360092:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff8b1006026300 x1855007612789376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078259.360108:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612789376/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078259.360116:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078259.360120:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078259.360122:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612789376:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078259.360128:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789504:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078259.360144:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078259.360152:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff8b1006026a00 x1855007612789504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078259.360164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612789504/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078259.360169:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078259.360171:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078259.360173:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789504:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078259.360193:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078259.360204:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078259.360234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff8b1006024000 x1855007612789632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078259.360243:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612789632/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078259.360248:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078259.360250:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078259.360252:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078259.360256:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789760:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078259.360272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078259.360283:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff8b1006025880 x1855007612789760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078259.360294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612789760/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078259.360307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078259.360310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078259.360314:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612789760:192.168.204.102@tcp:8:kworker.0 00010000:00010000:0.0:1769078260.255816:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078260.255817:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078261.279771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078261.279840:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078262.303761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078262.303830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078263.327747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078263.327815:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078264.351702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078264.351769:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078264.479789:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.479804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078264.479818:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.479830:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5252 00000100:00080000:0.0:1769078264.479844:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.479851:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078264.479896:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612789888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.479935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.479939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078264.479941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.479946:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5252 00000100:00080000:0.0:1769078264.479949:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.479952:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078264.479962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078264.479974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612790016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.479982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.479984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078264.479986:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.479989:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5252 00000100:00080000:0.0:1769078264.479993:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078264.479993:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612789888:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078264.479996:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078264.480000:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078264.480014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612790144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.480023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.480026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078264.480028:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.480033:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5252 00000100:00080000:0.0:1769078264.480037:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.480041:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078264.480046:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078264.480057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612790272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078264.480084:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078264.480151:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff8b1006026680 x1855007612789888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078264.480172:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612789888/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078264.480184:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078264.480189:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078264.480192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612789888:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078264.480217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790016:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078264.480237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078264.480249:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff8b1006025180 x1855007612790016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078264.480261:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612790016/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078264.480270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078264.480273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078264.480276:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790016:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078264.480282:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078264.480298:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078264.480307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff8b1006027800 x1855007612790144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078264.480326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612790144/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078264.480337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078264.480341:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078264.480344:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078264.480349:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790272:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078264.480366:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078264.480376:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff8b1006027100 x1855007612790272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078264.480387:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612790272/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078264.480395:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078264.480398:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078264.480401:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790272:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078265.375747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078265.375800:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078266.399750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078266.399756:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078267.423826:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078267.423826:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078268.447775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078268.447848:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078269.471786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078269.471820:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078269.599711:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078269.599721:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078269.599725:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078269.599735:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5257 00000100:00080000:1.0:1769078269.599746:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078269.599753:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078269.599785:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135c00 x1855007612790400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078269.599806:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078269.599809:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078269.599810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078269.599813:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5257 00000100:00080000:1.0:1769078269.599817:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078269.599819:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078269.599826:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078269.599832:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612790400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078269.599836:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137b80 x1855007612790528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078269.599845:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078269.599847:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078269.599848:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078269.599851:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5257 00000100:00080000:1.0:1769078269.599853:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078269.599855:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078269.599858:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078269.599865:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136a00 x1855007612790656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078269.599879:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078269.599882:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078269.599883:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078269.599887:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5257 00000800:00000100:0.0:1769078269.599891:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:1.0:1769078269.599893:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078269.599900:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078269.599905:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078269.599917:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612790784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078269.599936:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff8b1006135c00 x1855007612790400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.599950:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135c00 x1855007612790400/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.599957:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.599959:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078269.599962:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612790400:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078269.599979:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790528:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078269.599993:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078269.600001:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff8b1006137b80 x1855007612790528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.600008:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137b80 x1855007612790528/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.600012:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.600015:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078269.600017:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790528:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078269.600020:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790656:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078269.600030:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078269.600035:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff8b1006136a00 x1855007612790656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.600061:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136a00 x1855007612790656/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.600066:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.600068:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078269.600070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078269.600074:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790784:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078269.600086:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078269.600093:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff8b1006135500 x1855007612790784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.600099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612790784/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.600104:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.600105:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078269.600107:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612790784:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078270.495787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078270.495866:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078271.519866:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078271.519880:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078272.543870:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078272.543933:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078273.567726:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078273.567841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078274.591841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078274.591847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078274.719753:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078274.719767:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078274.719772:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078274.719784:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5262 00000100:00080000:3.0:1769078274.719798:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078274.719805:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078274.719852:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612790912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078274.719883:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078274.719887:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078274.719889:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078274.719894:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5262 00000100:00080000:3.0:1769078274.719897:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078274.719900:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078274.719912:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078274.719924:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612791040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078274.719936:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078274.719939:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078274.719943:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078274.719945:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612790912:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078274.719949:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5262 00000100:00080000:3.0:1769078274.719974:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078274.719979:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078274.719985:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078274.719997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612791168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078274.720007:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078274.720010:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078274.720011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078274.720015:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5262 00000100:00080000:3.0:1769078274.720019:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078274.720023:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078274.720027:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078274.720039:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078274.720040:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612791296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078274.720093:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff8b10037df800 x1855007612790912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.720113:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612790912/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.720121:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.720124:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078274.720127:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612790912:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078274.720148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791040:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078274.720167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078274.720179:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff8b10037dc700 x1855007612791040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.720191:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612791040/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.720200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.720203:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078274.720207:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791040:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078274.720213:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078274.720233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078274.720243:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff8b10037dce00 x1855007612791168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.720256:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612791168/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.720263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.720267:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078274.720270:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078274.720275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791296:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078274.720290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078274.720297:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff8b10037df480 x1855007612791296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.720304:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612791296/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.720309:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.720311:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078274.720313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791296:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078275.615720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078275.615841:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078276.639998:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078276.640404:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078277.663757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078277.663833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078278.687788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078278.687875:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078279.711777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078279.711831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078279.841834:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078279.841850:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078279.841855:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078279.841877:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5267 00000100:00080000:1.0:1769078279.841890:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078279.841897:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078279.841939:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612791424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078279.841974:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078279.841977:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078279.841979:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078279.841982:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5267 00000100:00080000:1.0:1769078279.841984:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078279.841987:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078279.841997:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078279.842007:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612791552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078279.842014:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078279.842017:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078279.842018:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078279.842020:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5267 00000100:00080000:1.0:1769078279.842022:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078279.842024:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078279.842026:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078279.842032:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612791680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078279.842037:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078279.842040:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078279.842041:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078279.842043:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5267 00000100:00080000:1.0:1769078279.842045:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078279.842047:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078279.842050:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078279.842055:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612791808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078279.842154:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612791424:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769078279.842233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078279.842277:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff8b1006136d80 x1855007612791424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.842291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612791424/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.842299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.842302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078279.842305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612791424:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078279.842310:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791552:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078279.842329:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078279.842342:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff8b1006136680 x1855007612791552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.842352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612791552/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.842359:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.842362:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078279.842364:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791552:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078279.842369:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078279.842387:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078279.842396:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff8b1006137100 x1855007612791680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.842412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612791680/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.842420:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.842422:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078279.842425:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791680:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078279.842429:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791808:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078279.842444:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078279.842453:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff8b1006134e00 x1855007612791808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.842464:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612791808/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.842472:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.842475:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078279.842479:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612791808:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078280.735774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078280.735832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078281.759871:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078281.759874:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078282.783865:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078282.783919:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078283.807768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078283.809495:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078284.831749:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078284.831836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078284.960330:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.960345:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078284.960350:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.960362:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5273 00000100:00080000:3.0:1769078284.960376:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.960383:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078284.960428:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612791936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.960463:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.960467:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078284.960469:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.960474:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5273 00000100:00080000:3.0:1769078284.960478:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.960482:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078284.960492:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078284.960505:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612792064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.960513:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.960517:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078284.960518:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.960522:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5273 00000100:00080000:3.0:1769078284.960525:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.960528:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078284.960532:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078284.960540:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612792192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.960553:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.960557:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078284.960558:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.960561:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5273 00000100:00080000:3.0:1769078284.960564:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.960567:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078284.960572:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078284.960579:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612792320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078284.960728:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612791936:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769078284.961041:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078284.961083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff8b10037dea00 x1855007612791936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078284.961102:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612791936/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078284.961112:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078284.961116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078284.961120:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612791936:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078284.961127:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792064:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078284.961880:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078284.961896:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff8b10037df480 x1855007612792064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078284.961909:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612792064/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078284.961918:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078284.961921:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078284.961924:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792064:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078284.961931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078284.961951:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078284.961963:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff8b10037dce00 x1855007612792192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078284.961974:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612792192/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078284.961987:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078284.961991:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078284.961994:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792192:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078284.962001:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792320:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078284.962020:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078284.962031:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff8b10037dc700 x1855007612792320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078284.962041:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612792320/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078284.962048:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078284.962051:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078284.962054:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792320:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078285.855781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078285.855852:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078286.879780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078286.879867:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078287.903790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078287.903850:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078288.927833:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078288.927860:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078289.951784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078289.951846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078290.079772:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.079782:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078290.079786:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.079795:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5278 00000100:00080000:3.0:1769078290.079806:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.079812:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078290.079847:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612792448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.079876:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.079880:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078290.079882:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.079885:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5278 00000100:00080000:3.0:1769078290.079888:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.079891:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078290.079899:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078290.079910:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612792576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.079917:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.079919:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078290.079920:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.079923:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5278 00000100:00080000:3.0:1769078290.079925:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.079927:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078290.079930:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078290.079941:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612792448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078290.079946:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612792704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.079962:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.079964:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078290.079965:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.079968:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5278 00000100:00080000:3.0:1769078290.079971:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.079973:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078290.079976:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078290.079983:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612792832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078290.080039:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078290.080102:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff8b10037df800 x1855007612792448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078290.080123:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612792448/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078290.080134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078290.080143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078290.080150:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612792448:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078290.080162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792576:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078290.080192:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078290.080206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff8b10037ddc00 x1855007612792576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078290.080218:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612792576/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078290.080224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078290.080227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078290.080230:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792576:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078290.080263:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078290.080279:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078290.080287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff8b10037df100 x1855007612792704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078290.080295:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612792704/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078290.080301:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078290.080303:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078290.080305:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078290.080310:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792832:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078290.080321:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078290.080330:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff8b10037dd880 x1855007612792832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078290.080337:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612792832/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078290.080342:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078290.080350:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078290.080352:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612792832:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078290.975732:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078290.975810:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078291.999811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078292.000027:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078293.023718:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078293.023794:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078294.047745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078294.047817:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078295.071731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078295.071785:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078295.199804:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078295.199820:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078295.199826:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078295.199838:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5283 00000100:00080000:1.0:1769078295.199853:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078295.199862:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078295.199908:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134700 x1855007612792960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078295.199939:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078295.199942:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078295.199944:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078295.199948:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5283 00000100:00080000:1.0:1769078295.199952:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078295.199956:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078295.199967:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078295.199981:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137480 x1855007612793088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078295.199991:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078295.199995:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612792960:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078295.199995:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078295.200000:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078295.200007:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5283 00000100:00080000:1.0:1769078295.200011:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078295.200014:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078295.200020:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078295.200031:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134a80 x1855007612793216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078295.200047:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078295.200050:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078295.200052:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078295.200056:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5283 00000100:00080000:1.0:1769078295.200062:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078295.200066:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078295.200071:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078295.200085:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006134e00 x1855007612793344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078295.200115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078295.200144:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078295.200165:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078296.095788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078296.095819:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078297.119706:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078297.119790:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078298.143780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078298.143841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769078298.271885:0:2351:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:2.0:1769078298.271921:0:2351:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769078298.271935:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078298.271973:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078298.271988:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:2.0:1769078298.271998:0:2351:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:0.0:1769078298.272706:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078298] req@ffff8b1006137480 x1855007612793088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078298.272754:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137480 x1855007612793088/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078298.272769:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078298.272776:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078298.272781:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793088:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078298.272792:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078298] req@ffff8b1006134a80 x1855007612793216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078298.272807:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134a80 x1855007612793216/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078298.272816:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078298.272820:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078298.272823:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078298.272837:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078298] req@ffff8b1006134e00 x1855007612793344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078298.272855:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134e00 x1855007612793344/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078298.272864:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078298.272870:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078298.272876:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793344:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078298.272885:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078298] req@ffff8b1006134700 x1855007612792960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078298.272899:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006134700 x1855007612792960/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078298.272905:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078298.272909:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078298.272912:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006134700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612792960:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078299.167740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078299.167796:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078300.191777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078300.191857:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078300.319866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078300.319880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078300.319885:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078300.319898:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5288 00000100:00080000:0.0:1769078300.319911:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078300.319920:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078300.319964:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612793472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078300.320050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612793472:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078300.320153:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078300.320185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078300.320190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078300.320192:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078300.320204:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5288 00000100:00080000:0.0:1769078300.320208:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078300.320213:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078300.320224:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078300.320237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612793600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078300.321316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff8b1006027100 x1855007612793472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.321341:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612793472/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.321353:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.321358:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078300.321362:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612793472:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078300.321396:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793600:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078300.321435:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078300.321451:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff8b1006027800 x1855007612793600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.321463:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612793600/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.321471:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.321474:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078300.321478:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793600:192.168.204.102@tcp:38:kworker.0 00000100:00080000:0.0:1769078300.321570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078300.321574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078300.321577:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078300.321591:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5288 00000100:00080000:0.0:1769078300.321596:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078300.321602:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078300.321610:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078300.321628:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612793728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078300.321669:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078300.321673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078300.321675:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078300.321679:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5288 00000100:00080000:0.0:1769078300.321682:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078300.321688:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078300.321693:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078300.321703:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612793856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078300.322558:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078300.322597:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078300.322621:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff8b1006025180 x1855007612793728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.322635:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612793728/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.322661:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.322664:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078300.322668:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078300.322674:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793856:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078300.322692:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078300.322702:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff8b1006026680 x1855007612793856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.322713:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612793856/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.322727:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.322730:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078300.322734:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612793856:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078301.215730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078301.215791:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078302.239770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078302.239901:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078303.263827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078303.263843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078304.287774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078304.287839:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078305.311751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078305.311812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078305.439793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078305.439810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078305.439815:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078305.439828:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5293 00000100:00080000:0.0:1769078305.439844:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078305.439853:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078305.439904:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612793984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078305.439941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078305.439945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078305.439947:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078305.439952:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5293 00000100:00080000:0.0:1769078305.439956:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078305.439960:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078305.439972:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078305.439986:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612794112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078305.439997:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612793984:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078305.440000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078305.440006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078305.440008:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078305.440013:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5293 00000100:00080000:0.0:1769078305.440019:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078305.440024:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078305.440030:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078305.440047:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612794240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078305.440057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078305.440060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078305.440063:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078305.440067:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5293 00000100:00080000:0.0:1769078305.440071:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769078305.440077:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078305.440080:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078305.440087:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078305.440101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612794368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078305.440129:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff8b1006025880 x1855007612793984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078305.440161:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612793984/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078305.440172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078305.440177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078305.440181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612793984:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078305.440208:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794112:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078305.440232:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078305.440242:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff8b1006024000 x1855007612794112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078305.440250:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612794112/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078305.440256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078305.440258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078305.440268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794112:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078305.440276:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078305.440296:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078305.440307:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff8b1006026a00 x1855007612794240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078305.440320:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612794240/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078305.440328:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078305.440332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078305.440335:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078305.440341:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794368:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078305.440361:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078305.440371:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff8b1006026300 x1855007612794368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078305.440383:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612794368/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078305.440391:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078305.440394:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078305.440397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794368:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078306.335766:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078306.335819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078307.359769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078307.359825:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078308.383770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078308.383824:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078309.407774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078309.407843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078310.431851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078310.431852:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:0.0:1769078310.559894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078310.559908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078310.559913:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078310.559926:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5298 00000100:00080000:0.0:1769078310.559940:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078310.559948:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078310.559995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612794496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078310.560036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078310.560040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078310.560043:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078310.560048:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5298 00000100:00080000:0.0:1769078310.560052:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078310.560056:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078310.560067:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078310.560081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612794624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078310.560085:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612794496:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078310.560100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078310.560103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078310.560105:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078310.560110:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5298 00000100:00080000:0.0:1769078310.560114:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078310.560118:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078310.560123:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078310.560135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612794752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078310.560144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078310.560147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078310.560149:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078310.560153:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5298 00000100:00080000:0.0:1769078310.560164:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078310.560168:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078310.560172:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078310.560176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078310.560187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612794880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078310.560217:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078310] req@ffff8b1006026d80 x1855007612794496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078310.560239:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612794496/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078310.560251:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078310.560256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078310.560261:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612794496:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078310.560291:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794624:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078310.560312:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078310.560324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078310] req@ffff8b1006027b80 x1855007612794624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078310.560332:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612794624/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078310.560338:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078310.560340:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078310.560342:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794624:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078310.560372:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078310.560389:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078310.560400:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078310] req@ffff8b1006025f80 x1855007612794752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078310.560412:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612794752/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078310.560430:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078310.560434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078310.560437:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078310.560443:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794880:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078310.560463:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078310.560474:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078310] req@ffff8b1006024a80 x1855007612794880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078310.560483:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612794880/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078310.560489:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078310.560491:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078310.560493:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612794880:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078311.455761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078311.455842:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078312.479759:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078312.479798:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078313.503774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078313.503824:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078314.527740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078314.527780:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078315.551774:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078315.551849:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078315.679725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.679735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078315.679740:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.679749:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5303 00000100:00080000:0.0:1769078315.679760:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.679767:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078315.679804:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612795008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078315.679830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.679834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078315.679836:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.679840:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5303 00000100:00080000:0.0:1769078315.679844:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.679849:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078315.679858:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078315.679871:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612795136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078315.679880:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612795008:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078315.679885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.679890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078315.679893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.679898:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5303 00000100:00080000:0.0:1769078315.679902:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.679906:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078315.679912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078315.679924:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612795264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078315.679934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.679937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078315.679940:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769078315.679942:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078315.679947:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5303 00000100:00080000:0.0:1769078315.679952:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.679957:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078315.679962:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078315.679974:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612795392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078315.679976:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff8b1006024700 x1855007612795008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.680007:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612795008/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.680015:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.680019:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078315.680021:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612795008:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078315.680039:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795136:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078315.680054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078315.680062:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff8b1006024e00 x1855007612795136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.680074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612795136/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.680082:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.680085:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078315.680088:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795136:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078315.680094:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078315.680112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078315.680122:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff8b1006024380 x1855007612795264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.680129:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612795264/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.680134:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.680136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078315.680138:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078315.680141:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795392:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078315.680154:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078315.680159:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff8b1006027480 x1855007612795392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.680167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612795392/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.680171:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.680173:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078315.680174:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795392:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078316.575753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078316.575814:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078317.599822:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078317.599866:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078318.623720:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078318.623785:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078319.647729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078319.647783:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078320.671702:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078320.671767:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078320.799737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.799747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078320.799752:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.799760:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5308 00000100:00080000:0.0:1769078320.799770:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.799777:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078320.799813:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612795520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.799833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.799836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078320.799837:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.799839:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5308 00000100:00080000:0.0:1769078320.799842:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.799844:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078320.799850:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078320.799858:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612795648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.799865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.799867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078320.799868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.799870:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5308 00000100:00080000:0.0:1769078320.799872:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.799876:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078320.799880:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078320.799880:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612795520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078320.799887:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612795776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.799894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.799896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078320.799897:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.799899:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5308 00000100:00080000:0.0:1769078320.799902:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.799904:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078320.799907:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078320.799913:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612795904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078320.799939:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078320.799965:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff8b1006027480 x1855007612795520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.799978:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612795520/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.799985:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.799988:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078320.799990:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612795520:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078320.799995:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795648:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078320.800015:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078320.800022:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff8b1006024380 x1855007612795648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.800029:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612795648/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.800038:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.800040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078320.800042:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795648:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078320.800059:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078320.800069:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078320.800074:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff8b1006024e00 x1855007612795776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.800086:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612795776/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.800094:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.800096:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078320.800098:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078320.800101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795904:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078320.800112:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078320.800120:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff8b1006024700 x1855007612795904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.800131:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612795904/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.800138:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.800141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078320.800145:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612795904:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078321.695713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078321.695795:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078322.719753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078322.719802:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078323.743767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078323.743840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078324.767799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078324.767833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078325.791750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078325.791830:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078325.919757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078325.919764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078325.919767:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078325.919775:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5314 00000100:00080000:2.0:1769078325.919785:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078325.919791:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078325.919820:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612796032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078325.919844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078325.919848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078325.919849:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078325.919853:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5314 00000100:00080000:2.0:1769078325.919857:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078325.919860:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078325.919867:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078325.919878:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612796160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078325.919887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078325.919889:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612796032:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078325.919890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078325.919893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078325.919902:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5314 00000100:00080000:2.0:1769078325.919905:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078325.919908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078325.919912:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078325.919920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612796288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078325.919926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078325.919928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078325.919929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078325.919931:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5314 00000100:00080000:2.0:1769078325.919933:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078325.919935:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078325.919939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078325.919944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612796416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078325.919961:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078325.920005:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff8b1005d5b800 x1855007612796032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078325.920029:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612796032/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078325.920040:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078325.920045:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078325.920050:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612796032:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078325.920077:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796160:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078325.920118:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078325.920133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff8b1005d58a80 x1855007612796160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078325.920154:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612796160/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078325.920163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078325.920166:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078325.920169:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796160:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078325.920176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078325.920195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078325.920206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff8b1005d58e00 x1855007612796288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078325.920228:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612796288/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078325.920235:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078325.920238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078325.920241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078325.920247:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796416:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078325.920278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078325.920294:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff8b1005d5a300 x1855007612796416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078325.920309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612796416/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078325.920322:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078325.920326:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078325.920330:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796416:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078326.815770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078326.815811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078327.839788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078327.839825:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078328.863750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078328.863822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078329.887727:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078329.887786:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078330.911739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078330.911802:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078331.039773:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.039783:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078331.039787:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.039796:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5319 00000100:00080000:3.0:1769078331.039806:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.039813:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078331.039849:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612796544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.039878:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.039882:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078331.039883:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.039887:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5319 00000100:00080000:3.0:1769078331.039891:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.039895:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078331.039904:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078331.039916:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612796672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.039923:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.039925:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078331.039927:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.039929:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5319 00000100:00080000:3.0:1769078331.039931:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.039934:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078331.039937:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078331.039943:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612796800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078331.039948:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078331.039950:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078331.039951:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078331.039953:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5319 00000100:00080000:3.0:1769078331.039955:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078331.039956:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078331.039959:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078331.039963:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612796928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078331.040102:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612796544:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769078331.040159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078331.040193:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff8b10037dc000 x1855007612796544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078331.040207:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612796544/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078331.040214:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078331.040218:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078331.040220:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612796544:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078331.040225:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796672:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078331.040241:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078331.040250:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff8b10037dd500 x1855007612796672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078331.040262:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612796672/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078331.040288:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078331.040291:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078331.040294:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796672:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078331.040300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078331.040316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078331.040324:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff8b10037ddf80 x1855007612796800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078331.040331:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612796800/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078331.040337:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078331.040339:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078331.040340:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796800:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078331.040344:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796928:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078331.040353:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078331.040359:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078331/real 1769078331] req@ffff8b10037de300 x1855007612796928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078331.040365:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612796928/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078331.040370:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078331.040371:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078331.040373:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612796928:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078331.935758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078331.935764:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078332.959773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078332.959850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078333.983750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078333.983808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078335.007748:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078335.007805:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078336.031723:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078336.031792:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078336.159800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.159811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078336.159815:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.159825:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5324 00000100:00080000:2.0:1769078336.159836:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.159842:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078336.159875:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612797056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.159903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.159907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078336.159909:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.159913:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5324 00000100:00080000:2.0:1769078336.159928:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.159933:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078336.159942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078336.159953:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612797184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.159966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078336.159970:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612797056:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078336.159971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078336.159976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.159986:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5324 00000100:00080000:2.0:1769078336.159989:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.159992:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078336.159997:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078336.160014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612797312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.160023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.160027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078336.160029:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.160033:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5324 00000100:00080000:2.0:1769078336.160036:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.160040:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078336.160045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078336.160053:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612797440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078336.160072:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078336.160121:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff8b1005d59180 x1855007612797056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078336.160143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612797056/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078336.160152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078336.160156:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078336.160159:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612797056:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078336.160185:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797184:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078336.160203:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078336.160212:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff8b1005d58000 x1855007612797184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078336.160220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612797184/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078336.160225:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078336.160227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078336.160229:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797184:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078336.160234:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797312:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078336.160246:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078336.160252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff8b1005d5a300 x1855007612797312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078336.160258:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612797312/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078336.160263:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078336.160265:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078336.160267:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078336.160270:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797440:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078336.160280:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078336.160285:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff8b1005d58e00 x1855007612797440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078336.160291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612797440/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078336.160296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078336.160298:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078336.160299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797440:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078337.055771:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078337.055839:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078338.079716:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078338.079792:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078339.103758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078339.103823:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078340.127714:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078340.127801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078341.151760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078341.151810:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078341.279819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078341.279847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078341.279859:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078341.279871:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5329 00000100:00080000:3.0:1769078341.279889:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078341.279897:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078341.279966:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612797568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078341.280005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078341.280019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078341.280022:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078341.280028:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5329 00000100:00080000:3.0:1769078341.280032:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078341.280037:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078341.280050:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078341.280066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612797696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078341.280071:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612797568:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078341.280101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078341.280105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078341.280107:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078341.280113:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5329 00000100:00080000:3.0:1769078341.280118:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078341.280123:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078341.280128:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078341.280149:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612797824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078341.280150:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078341.280170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078341.280174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078341.280178:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078341.280187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff8b10037ded80 x1855007612797568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078341.280191:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5329 00000100:00080000:3.0:1769078341.280203:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078341.280214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612797568/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078341.280216:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078341.280231:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1769078341.280232:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078341.280239:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078341.280242:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612797568:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078341.280256:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612797952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078341.280264:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797696:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078341.280290:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078341.280301:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff8b10037dca80 x1855007612797696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078341.280309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612797696/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078341.280314:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078341.280318:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078341.280321:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797696:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078341.280325:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078341.280336:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078341.280342:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff8b10037dc380 x1855007612797824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078341.280352:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612797824/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078341.280357:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078341.280358:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078341.280360:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078341.280368:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797952:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078341.280383:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078341.280393:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff8b10037dd180 x1855007612797952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078341.280403:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612797952/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078341.280411:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078341.280414:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078341.280417:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612797952:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078342.175707:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078342.175741:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078343.199772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078343.199965:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078344.223772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078344.223844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078345.247787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078345.247841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078346.271810:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078346.271913:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078346.399763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078346.399774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078346.399778:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078346.399787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5334 00000100:00080000:3.0:1769078346.399797:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078346.399803:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078346.399837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612798080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078346.399862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078346.399865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078346.399867:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078346.399869:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5334 00000100:00080000:3.0:1769078346.399872:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078346.399875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078346.399881:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078346.399891:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612798208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078346.399899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078346.399901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078346.399902:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078346.399904:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5334 00000100:00080000:3.0:1769078346.399906:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078346.399908:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078346.399911:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078346.399918:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612798336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078346.399923:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612798080:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078346.399926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078346.399931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078346.399933:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078346.399935:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5334 00000100:00080000:3.0:1769078346.399937:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078346.399939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078346.399942:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078346.399949:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612798464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078346.399982:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078346.400023:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff8b10037de680 x1855007612798080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078346.400046:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612798080/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078346.400065:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078346.400069:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078346.400072:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612798080:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078346.400079:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798208:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078346.400096:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078346.400106:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff8b10037dd180 x1855007612798208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078346.400116:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612798208/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078346.400124:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078346.400127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078346.400130:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798208:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078346.400152:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078346.400171:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078346.400181:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff8b10037dc380 x1855007612798336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078346.400192:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612798336/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078346.400205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078346.400209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078346.400212:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798336:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078346.400217:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798464:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078346.400235:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078346.400246:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff8b10037dca80 x1855007612798464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078346.400257:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612798464/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078346.400265:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078346.400268:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078346.400271:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798464:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078347.295799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078347.295841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078348.319745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078348.319812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078349.343744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078349.343801:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078350.367800:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078350.367866:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078351.391768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078351.391811:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078351.519797:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.519810:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078351.519815:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.519825:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5339 00000100:00080000:3.0:1769078351.519843:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.519850:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078351.519893:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612798592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078351.519918:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.519920:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078351.519922:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.519925:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5339 00000100:00080000:3.0:1769078351.519927:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.519929:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078351.519938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078351.519946:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612798720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078351.519953:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.519957:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078351.519959:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.519963:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5339 00000100:00080000:3.0:1769078351.519967:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.519971:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078351.519985:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078351.519997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612798848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078351.520008:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612798592:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078351.520012:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.520021:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078351.520027:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.520032:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5339 00000100:00080000:3.0:1769078351.520035:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.520038:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078351.520042:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078351.520051:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612798976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078351.520101:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078351.520136:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff8b10037ded80 x1855007612798592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078351.520153:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612798592/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078351.520170:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078351.520174:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078351.520178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612798592:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078351.520186:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798720:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078351.520204:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078351.520214:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff8b10037de300 x1855007612798720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078351.520226:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612798720/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078351.520233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078351.520236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078351.520240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798720:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078351.520261:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078351.520276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078351.520285:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff8b10037ddf80 x1855007612798848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078351.520297:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612798848/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078351.520304:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078351.520307:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078351.520310:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078351.520316:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798976:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078351.520331:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078351.520339:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff8b10037dd500 x1855007612798976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078351.520350:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612798976/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078351.520357:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078351.520360:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078351.520363:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612798976:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078352.415783:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078352.415890:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078353.439744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078353.439771:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078354.463756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078354.463828:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078355.487740:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078355.487779:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078356.511758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078356.511916:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078356.639781:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078356.639792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078356.639796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078356.639805:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5344 00000100:00080000:2.0:1769078356.639816:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078356.639823:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078356.639859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612799104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078356.639888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078356.639892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078356.639895:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078356.639898:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5344 00000100:00080000:2.0:1769078356.639902:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078356.639906:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078356.639915:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078356.639927:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612799232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078356.639931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612799104:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078356.639945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078356.639948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078356.639951:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078356.639955:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5344 00000100:00080000:2.0:1769078356.639959:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078356.639962:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078356.639967:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078356.639978:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612799360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078356.639987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078356.639991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1769078356.639994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:2.0:1769078356.639995:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078356.640001:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5344 00000100:00080000:2.0:1769078356.640005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078356.640009:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078356.640014:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078356.640025:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612799488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078356.640026:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff8b1005d58a80 x1855007612799104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078356.640056:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612799104/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078356.640063:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078356.640067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078356.640070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612799104:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078356.640087:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799232:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078356.640100:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078356.640107:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff8b1005d5b800 x1855007612799232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078356.640114:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612799232/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078356.640119:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078356.640121:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078356.640122:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799232:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078356.640126:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078356.640135:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078356.640140:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff8b1005d59500 x1855007612799360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078356.640147:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612799360/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078356.640152:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078356.640153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078356.640155:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078356.640158:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799488:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078356.640167:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078356.640172:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff8b1005d5b480 x1855007612799488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078356.640182:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612799488/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078356.640187:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078356.640188:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078356.640190:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799488:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078357.535729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078357.535786:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078358.559756:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078358.559828:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078359.583696:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078359.583738:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078360.607809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078360.607832:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078361.631784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078361.631884:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078361.759977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078361.759990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078361.759995:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078361.760006:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5349 00000100:00080000:3.0:1769078361.760018:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078361.760026:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078361.760060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612799616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078361.760086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078361.760090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078361.760092:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078361.760097:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5349 00000100:00080000:3.0:1769078361.760102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078361.760106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078361.760116:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078361.760118:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612799616:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078361.760132:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078361.760145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078361.760148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078361.760150:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078361.760155:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5349 00000100:00080000:3.0:1769078361.760161:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078361.760166:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078361.760181:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078361.760195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612799872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078361.760207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078361.760211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078361.760214:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078361.760219:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5349 00000100:00100000:1.0:1769078361.760222:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799744:192.168.204.102@tcp:38:kworker.0 00000100:00080000:3.0:1769078361.760226:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769078361.760254:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799872:192.168.204.102@tcp:8:kworker.0 00000100:00080000:3.0:1769078361.760305:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078361.760312:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078361.760327:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612800000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078361.760405:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078362.655761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078362.655822:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078363.679769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078363.679830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078364.703772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078364.703833:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769078364.839520:0:2353:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:3.0:1769078364.839559:0:2353:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078364.839574:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078364.839612:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078364.839620:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:3.0:1769078364.839628:0:2353:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769078364.839675:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078364] req@ffff8b10037ddc00 x1855007612800000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078364.839699:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612800000/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078364.839715:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078364.839719:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078364.839723:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800000:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078364.839730:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078364] req@ffff8b10037dd880 x1855007612799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078364.839753:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612799744/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078364.839760:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078364.839764:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078364.839767:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799744:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078364.839775:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078364] req@ffff8b10037df100 x1855007612799872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078364.839785:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612799872/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078364.839792:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078364.839795:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078364.839798:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612799872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078364.839803:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078364] req@ffff8b10037dc000 x1855007612799616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078364.839815:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612799616/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078364.839822:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078364.839825:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078364.839828:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612799616:192.168.204.102@tcp:250:kworker.0 00010000:00010000:0.0:1769078365.727801:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078365.727902:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078366.751730:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078366.751748:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078366.879745:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078366.879755:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078366.879760:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078366.879771:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5354 00000100:00080000:3.0:1769078366.879781:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078366.879796:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078366.879829:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612800128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078366.879852:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078366.879856:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078366.879858:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078366.879862:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5354 00000100:00080000:3.0:1769078366.879866:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078366.879871:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078366.879880:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078366.879893:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612800256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078366.879900:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612800128:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078366.879916:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078366.879924:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078366.879926:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078366.879932:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5354 00000100:00080000:3.0:1769078366.879936:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078366.879940:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078366.879945:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078366.879960:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612800384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078366.879970:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078366.879974:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078366.879976:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078366.879980:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5354 00000100:00080000:3.0:1769078366.879985:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078366.879988:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078366.879994:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078366.880000:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078366.880010:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612800512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078366.880047:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff8b10037df800 x1855007612800128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.880074:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612800128/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.880088:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.880094:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078366.880099:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612800128:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078366.880132:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800256:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078366.880155:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078366.880167:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff8b10037dc700 x1855007612800256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.880180:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612800256/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.880188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.880192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078366.880195:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800256:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078366.880202:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078366.880220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078366.880228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff8b10037dce00 x1855007612800384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.880240:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612800384/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.880247:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.880251:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078366.880254:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078366.880262:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800512:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078366.880278:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078366.880287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff8b10037df480 x1855007612800512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.880299:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612800512/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.880307:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.880310:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078366.880313:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800512:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078367.775816:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078367.775823:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078368.799725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078368.799776:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078369.823764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078369.823812:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078370.847747:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078370.848298:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078371.871809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078371.872198:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078371.999817:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078371.999832:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078371.999838:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078371.999853:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5360 00000100:00080000:0.0:1769078371.999868:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078371.999875:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078371.999919:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612800640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078371.999957:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078371.999962:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078371.999964:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078371.999970:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5360 00000100:00080000:0.0:1769078371.999973:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078371.999977:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078371.999987:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078371.999997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612800768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.000005:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.000009:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078372.000011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.000015:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5360 00000100:00080000:0.0:1769078372.000018:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.000022:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078372.000026:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078372.000036:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612800896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078372.000045:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078372.000048:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078372.000050:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078372.000054:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5360 00000100:00080000:0.0:1769078372.000057:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078372.000061:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078372.000065:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078372.000076:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612801024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078372.000162:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612800640:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769078372.000237:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078372.000270:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078372] req@ffff8b1006024a80 x1855007612800640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.000285:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612800640/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.000293:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.000296:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078372.000299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612800640:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078372.000305:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800768:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078372.000348:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078372.000357:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078372] req@ffff8b1006025f80 x1855007612800768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.000365:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612800768/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.000370:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.000372:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078372.000374:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800768:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078372.000378:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.000390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078372.000395:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078372] req@ffff8b1006027b80 x1855007612800896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.000402:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612800896/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.000407:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.000408:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078372.000410:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612800896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078372.000413:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801024:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078372.000424:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078372.000430:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078372] req@ffff8b1006026d80 x1855007612801024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078372.000437:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612801024/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078372.000442:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078372.000443:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078372.000445:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801024:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078372.895772:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078372.895850:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078373.919769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078373.919834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078374.944112:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078374.945562:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078375.967773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078375.967920:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078376.991745:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078376.991830:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078377.119774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.119796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078377.119801:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.119812:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5365 00000100:00080000:3.0:1769078377.119824:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.119830:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078377.119881:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612801152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.119907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.119910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078377.119911:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.119914:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5365 00000100:00080000:3.0:1769078377.119917:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.119920:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078377.119930:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078377.119945:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612801280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.119954:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612801152:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078377.119959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.119964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078377.119966:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.119971:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5365 00000100:00080000:3.0:1769078377.119976:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.119980:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078377.119985:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078377.119995:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612801408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.120006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.120008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078377.120010:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.120012:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5365 00000100:00080000:3.0:1769078377.120015:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.120019:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:2.0:1769078377.120020:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769078377.120023:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078377.120033:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612801536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.120058:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff8b10037dea00 x1855007612801152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.120071:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612801152/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.120079:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.120082:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078377.120085:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612801152:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078377.120104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801280:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078377.120118:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078377.120125:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff8b10037df480 x1855007612801280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.120133:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612801280/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.120139:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.120141:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078377.120143:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801280:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078377.120147:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.120157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078377.120162:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff8b10037dce00 x1855007612801408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.120169:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612801408/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.120174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.120176:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078377.120177:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078377.120180:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801536:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078377.120189:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078377.120195:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078377] req@ffff8b10037dc700 x1855007612801536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078377.120201:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612801536/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078377.120215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078377.120217:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078377.120219:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801536:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078378.015801:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078378.015899:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078379.039808:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078379.039894:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078380.063742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078380.063800:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078381.087719:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078381.087766:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078382.111821:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078382.111835:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078382.239729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078382.239737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078382.239741:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078382.239747:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5370 00000100:00080000:0.0:1769078382.239756:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078382.239761:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078382.239789:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612801664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078382.239812:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078382.239815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078382.239816:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078382.239818:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5370 00000100:00080000:0.0:1769078382.239820:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078382.239822:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078382.239829:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078382.239837:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078382.239848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078382.239849:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612801664:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078382.239852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078382.239856:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078382.239859:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5370 00000100:00080000:0.0:1769078382.239861:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078382.239863:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078382.239866:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078382.239877:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612801920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078382.239885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078382.239888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078382.239890:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078382.239893:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5370 00000100:00080000:0.0:1769078382.239899:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078382.239903:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078382.239908:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078382.239911:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078382.239921:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612802048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078382.239945:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff8b1006026300 x1855007612801664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.239960:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612801664/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.239967:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.239970:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078382.239973:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612801664:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078382.239990:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801792:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078382.240006:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078382.240013:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff8b1006026a00 x1855007612801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.240021:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612801792/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.240027:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.240029:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078382.240031:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801792:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078382.240034:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078382.240044:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078382.240049:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff8b1006024000 x1855007612801920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.240055:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612801920/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.240060:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.240062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078382.240064:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612801920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078382.240067:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802048:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078382.240075:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078382.240080:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff8b1006025880 x1855007612802048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.240086:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612802048/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.240091:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.240093:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078382.240095:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802048:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078383.135785:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078383.135840:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078384.159779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078384.159851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078385.183769:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078385.183862:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078386.207737:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078386.207814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078387.231794:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078387.231856:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078387.359791:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.359805:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078387.359811:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.359825:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5375 00000100:00080000:0.0:1769078387.359839:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.359848:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078387.359899:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612802176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078387.359932:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.359937:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078387.359939:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.359944:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5375 00000100:00080000:0.0:1769078387.359948:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.359952:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078387.359964:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078387.359979:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078387.359982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612802176:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078387.360005:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.360009:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078387.360011:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.360016:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5375 00000100:00080000:0.0:1769078387.360021:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.360025:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078387.360031:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078387.360043:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612802432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078387.360076:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:0.0:1769078387.360092:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.360096:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078387.360099:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.360105:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5375 00000100:00080000:0.0:1769078387.360109:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.360114:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078387.360117:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff8b1006026680 x1855007612802176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769078387.360121:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078387.360135:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612802176/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078387.360144:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612802560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078387.360144:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.360151:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078387.360158:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612802176:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078387.360183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802304:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078387.360199:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078387.360207:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff8b1006025180 x1855007612802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.360214:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612802304/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.360219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.360222:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078387.360225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802304:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078387.360228:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802432:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078387.360239:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078387.360244:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff8b1006027800 x1855007612802432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.360251:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612802432/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.360256:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.360258:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078387.360259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802432:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078387.360262:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802560:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078387.360272:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078387.360277:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff8b1006027100 x1855007612802560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.360284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612802560/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.360289:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.360291:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078387.360292:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802560:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078388.255788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078388.255858:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078389.279837:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078389.279846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078390.303712:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078390.303787:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078391.327814:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078391.327844:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078392.351733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078392.351808:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078392.479809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078392.479820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078392.479825:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078392.479834:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5380 00000100:00080000:3.0:1769078392.479845:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078392.479851:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078392.479897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612802688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078392.479920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078392.479924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078392.479926:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078392.479930:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5380 00000100:00080000:3.0:1769078392.479934:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078392.479938:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078392.479947:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078392.479959:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612802816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078392.479963:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612802688:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078392.479981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078392.479985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078392.479987:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078392.479991:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5380 00000100:00080000:3.0:1769078392.479995:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078392.479999:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078392.480004:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078392.480016:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612802944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078392.480024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078392.480028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078392.480029:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078392.480033:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5380 00000100:00080000:3.0:1769078392.480036:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078392.480038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00080000:3.0:1769078392.480043:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078392.480048:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078392.480059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612803072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078392.480072:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff8b10037df800 x1855007612802688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078392.480088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612802688/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078392.480097:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078392.480101:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078392.480105:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612802688:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078392.480125:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802816:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078392.480142:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078392.480150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff8b10037dc000 x1855007612802816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078392.480158:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612802816/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078392.480163:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078392.480165:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078392.480167:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802816:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078392.480170:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802944:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078392.480183:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078392.480189:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff8b10037df100 x1855007612802944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078392.480196:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612802944/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078392.480200:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078392.480201:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078392.480203:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612802944:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078392.480206:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803072:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078392.480215:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078392.480220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff8b10037dd880 x1855007612803072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078392.480227:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612803072/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078392.480231:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078392.480233:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078392.480234:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803072:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078393.375784:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078393.375998:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078394.399762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078394.399911:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078395.423770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078395.425224:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078396.447897:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078396.447915:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078397.471753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078397.471980:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078397.600033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.600041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078397.600044:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.600053:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5385 00000100:00080000:3.0:1769078397.600063:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.600070:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078397.600099:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612803200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.600127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.600131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078397.600133:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.600137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5385 00000100:00080000:3.0:1769078397.600141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.600144:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078397.600154:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078397.600166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612803328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.600176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.600179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078397.600181:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.600184:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5385 00000100:00080000:3.0:1769078397.600188:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.600192:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078397.600197:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078397.600206:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612803456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.600214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.600217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078397.600219:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.600222:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5385 00000100:00080000:3.0:1769078397.600225:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.600227:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078397.600232:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078397.600240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612803584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078397.600345:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612803200:192.168.204.102@tcp:250:kworker.0 00000800:00000100:2.0:1769078397.600398:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078397.600425:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff8b10037ddc00 x1855007612803200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078397.600441:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612803200/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078397.600449:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078397.600452:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078397.600456:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612803200:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078397.600462:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803328:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078397.600480:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078397.600490:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff8b10037dd500 x1855007612803328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078397.600499:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612803328/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078397.600505:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078397.600508:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078397.600511:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803328:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078397.600516:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803456:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078397.600531:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078397.600540:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff8b10037ddf80 x1855007612803456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078397.600555:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612803456/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078397.600561:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078397.600564:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078397.600566:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803456:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078397.600570:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803584:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078397.600584:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078397.600593:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff8b10037de300 x1855007612803584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078397.600604:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612803584/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078397.600611:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078397.600614:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078397.600617:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803584:192.168.204.102@tcp:8:kworker.0 00010000:00010000:3.0:1769078398.495886:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078398.495944:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078399.520044:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078399.520929:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078400.543803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078400.543886:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078401.567767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078401.567861:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078402.592439:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078402.592506:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078402.719784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078402.719813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078402.719819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078402.719832:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5390 00000100:00080000:3.0:1769078402.719846:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078402.719855:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078402.719900:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612803712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078402.719937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078402.719940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078402.719941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078402.719945:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5390 00000100:00080000:3.0:1769078402.719947:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078402.719951:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078402.719959:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078402.719969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612803840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078402.719976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078402.719979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078402.719980:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078402.719983:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5390 00000100:00080000:3.0:1769078402.719984:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078402.719986:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078402.719990:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078402.719994:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612803712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078402.719999:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612803968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078402.720009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078402.720011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078402.720012:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078402.720014:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5390 00000100:00080000:3.0:1769078402.720016:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078402.720019:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078402.720022:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078402.720029:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612804096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078402.720079:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078402.720123:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff8b10037ded80 x1855007612803712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.720143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612803712/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.720155:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.720159:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078402.720164:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612803712:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078402.720171:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803840:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078402.720193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078402.720206:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff8b10037dca80 x1855007612803840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.720219:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612803840/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.720233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.720236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078402.720240:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803840:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078402.720265:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803968:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078402.720287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078402.720298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff8b10037dc380 x1855007612803968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.720310:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612803968/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.720324:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.720328:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078402.720331:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612803968:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078402.720337:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804096:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078402.720356:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078402.720367:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff8b10037dd180 x1855007612804096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.720378:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612804096/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.720385:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.720389:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078402.720391:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804096:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078403.615851:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078403.617368:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078404.639799:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078404.640981:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078405.663803:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078405.667118:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078406.688033:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078406.691504:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078407.711813:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078407.712174:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078407.839793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.839806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078407.839812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.839824:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5395 00000100:00080000:3.0:1769078407.839838:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.839847:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078407.839897:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612804224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.839939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.839945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078407.839947:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.839951:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5395 00000100:00080000:3.0:1769078407.839955:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.839959:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078407.839971:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078407.839985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612804352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.839998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.840001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078407.840003:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078407.840008:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612804224:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078407.840010:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5395 00000100:00080000:3.0:1769078407.840018:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.840024:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078407.840029:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078407.840043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612804480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.840052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.840056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078407.840058:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.840062:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5395 00000100:00080000:3.0:1769078407.840066:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.840070:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078407.840075:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078407.840085:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612804608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078407.840106:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078407.840150:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff8b10037de680 x1855007612804224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.840170:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612804224/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.840182:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.840187:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078407.840192:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612804224:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078407.840218:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804352:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078407.840241:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078407.840254:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff8b10037dd180 x1855007612804352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.840272:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612804352/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.840281:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.840285:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078407.840289:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804352:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078407.840295:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804480:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078407.840316:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078407.840326:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff8b10037dc380 x1855007612804480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.840339:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612804480/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.840347:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.840351:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078407.840355:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804480:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078407.840368:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804608:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078407.840388:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078407.840399:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff8b10037dca80 x1855007612804608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.840411:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612804608/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.840419:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.840422:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078407.840426:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804608:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078408.735827:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078408.735936:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078409.761284:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078409.762179:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078410.783779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078410.784418:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078411.807791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078411.807862:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078412.831796:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078412.831941:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078412.959793:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078412.959805:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078412.959810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078412.959820:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5401 00000100:00080000:2.0:1769078412.959832:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078412.959839:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078412.959880:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612804736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078412.959905:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078412.959908:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078412.959909:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078412.959912:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5401 00000100:00080000:2.0:1769078412.959915:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078412.959927:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078412.959938:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078412.959953:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612804864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078412.959965:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612804736:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078412.959967:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078412.959975:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078412.959977:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078412.959981:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5401 00000100:00080000:2.0:1769078412.959984:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078412.959987:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078412.959992:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078412.960017:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612804992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078412.960027:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078412.960030:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078412.960032:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078412.960037:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5401 00000100:00080000:2.0:1769078412.960041:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078412.960046:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078412.960051:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078412.960054:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078412.960066:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612805120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078412.960106:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff8b1005d5bb80 x1855007612804736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078412.960127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612804736/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078412.960138:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078412.960143:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078412.960156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612804736:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078412.960187:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804864:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078412.960211:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078412.960223:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff8b1005d59880 x1855007612804864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078412.960235:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612804864/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078412.960252:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078412.960256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078412.960260:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804864:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078412.960266:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804992:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078412.960287:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078412.960298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff8b1005d5aa00 x1855007612804992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078412.960310:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612804992/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078412.960317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078412.960321:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078412.960324:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612804992:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078412.960338:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805120:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078412.960358:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078412.960369:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff8b1005d5b100 x1855007612805120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078412.960381:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612805120/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078412.960389:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078412.960392:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078412.960395:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805120:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078413.855872:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078413.856005:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078414.879804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078414.879919:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078415.903789:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078415.904000:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078416.927773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078416.927835:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078417.951781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078417.951853:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078418.079774:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.079782:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078418.079785:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.079793:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5406 00000100:00080000:1.0:1769078418.079802:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.079806:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:1.0:1769078418.079834:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006137100 x1855007612805248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.079854:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.079856:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078418.079857:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.079860:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5406 00000100:00080000:1.0:1769078418.079862:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.079864:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078418.079871:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078418.079879:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136680 x1855007612805376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.079885:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.079888:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078418.079889:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.079894:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5406 00000100:00100000:2.0:1769078418.079896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612805248:192.168.204.102@tcp:250:kworker.0 00000100:00080000:1.0:1769078418.079898:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.079905:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078418.079910:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078418.079922:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006136d80 x1855007612805504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.079932:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.079935:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078418.079937:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.079941:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5406 00000100:00080000:1.0:1769078418.079946:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.079950:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:1.0:1769078418.079954:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078418.079961:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078418.079966:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006135500 x1855007612805632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078418.079995:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff8b1006137100 x1855007612805248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.080012:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006137100 x1855007612805248/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.080021:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.080025:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078418.080028:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006137100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612805248:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078418.080048:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805376:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078418.080068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078418.080078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff8b1006136680 x1855007612805376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.080088:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136680 x1855007612805376/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.080095:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.080098:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078418.080100:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805376:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078418.080105:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805504:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.080121:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078418.080130:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff8b1006136d80 x1855007612805504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.080139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006136d80 x1855007612805504/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.080145:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.080148:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078418.080151:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006136d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805504:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.080155:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805632:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.080169:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078418.080177:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff8b1006135500 x1855007612805632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.080187:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006135500 x1855007612805632/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.080194:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.080197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078418.080200:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006135500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805632:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078418.975713:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078418.975753:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078419.999758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078419.999834:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078421.023725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078421.023769:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078422.047776:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078422.048918:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078423.071717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078423.071836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078423.200635:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.200671:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078423.200676:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.200688:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5411 00000100:00080000:3.0:1769078423.200711:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.200719:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078423.200765:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612805760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078423.200788:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.200791:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078423.200792:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.200794:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5411 00000100:00080000:3.0:1769078423.200797:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.200801:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078423.200809:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078423.200819:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612805888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078423.200830:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.200834:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078423.200836:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.200841:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5411 00000100:00080000:3.0:1769078423.200846:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.200851:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078423.200856:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078423.200871:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612806016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078423.200881:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612805760:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078423.200882:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.200888:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078423.200893:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.200902:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5411 00000100:00080000:3.0:1769078423.200906:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.200909:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078423.200912:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078423.200925:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612806144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078423.200969:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078423.201008:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff8b10037ded80 x1855007612805760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.201028:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612805760/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.201054:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.201059:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078423.201063:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612805760:192.168.204.102@tcp:250:kworker.0 00000100:00100000:2.0:1769078423.201071:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805888:192.168.204.102@tcp:38:kworker.0 00000800:00000100:2.0:1769078423.201092:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078423.201114:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff8b10037de300 x1855007612805888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.201127:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612805888/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.201135:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.201138:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078423.201141:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612805888:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078423.201176:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806016:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078423.201208:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078423.201220:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff8b10037ddf80 x1855007612806016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.201232:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612806016/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.201246:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.201259:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078423.201262:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806016:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078423.201269:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806144:192.168.204.102@tcp:8:kworker.0 00000800:00000100:2.0:1769078423.201288:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:2.0:1769078423.201299:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff8b10037dd500 x1855007612806144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.201310:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612806144/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.201519:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.201524:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078423.201528:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806144:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078424.095746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078424.095809:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078425.119788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078425.119878:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078426.143768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078426.143814:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078427.167773:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078427.167844:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078428.191809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078428.191841:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078428.319770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078428.319780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078428.319784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078428.319793:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5416 00000100:00080000:0.0:1769078428.319805:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078428.319811:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078428.319854:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027100 x1855007612806272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078428.319877:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078428.319880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078428.319881:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078428.319884:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5416 00000100:00080000:0.0:1769078428.319887:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078428.319890:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078428.319899:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078428.319908:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027800 x1855007612806400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078428.319916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078428.319918:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078428.319919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078428.319924:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5416 00000100:00100000:2.0:1769078428.319924:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612806272:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078428.319926:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078428.319932:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078428.319937:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078428.319944:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025180 x1855007612806528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078428.319948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078428.319951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078428.319952:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078428.319954:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5416 00000100:00080000:0.0:1769078428.319956:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078428.319958:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078428.319961:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078428.319969:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026680 x1855007612806656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078428.320068:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078428.320104:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078428.320130:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078429.215795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078429.215854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078430.239739:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078430.239800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078431.263758:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078431.263833:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769078431.391926:0:2352:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:1.0:1769078431.391982:0:2352:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769078431.392004:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078431.392054:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078431.392065:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:1.0:1769078431.392077:0:2352:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:2.0:1769078431.392115:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078431] req@ffff8b1006027800 x1855007612806400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078431.392152:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027800 x1855007612806400/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078431.392166:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078431.392172:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078431.392176:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806400:192.168.204.102@tcp:38:kworker.0 00000100:00100000:2.0:1769078431.392185:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078431] req@ffff8b1006025180 x1855007612806528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078431.392197:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025180 x1855007612806528/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078431.392205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078431.392209:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078431.392212:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806528:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078431.392219:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078431] req@ffff8b1006026680 x1855007612806656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078431.392231:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026680 x1855007612806656/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078431.392238:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078431.392242:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078431.392245:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806656:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078431.392252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078431] req@ffff8b1006027100 x1855007612806272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078431.392264:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027100 x1855007612806272/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078431.392270:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078431.392273:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:2.0:1769078431.392277:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612806272:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078432.287792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078432.287836:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078433.311833:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078433.311949:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078433.440483:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078433.440494:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078433.440499:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078433.440515:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5421 00000100:00080000:3.0:1769078433.440527:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078433.440534:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078433.440572:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612806784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078433.440615:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078433.440624:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078433.440626:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078433.440632:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5421 00000100:00080000:3.0:1769078433.440675:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078433.440681:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078433.440692:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078433.440708:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612806912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078433.440721:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078433.440725:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078433.440727:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078433.440732:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5421 00000100:00080000:3.0:1769078433.440737:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078433.440741:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078433.440747:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078433.440760:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612807040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078433.440769:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078433.440772:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078433.440774:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078433.440777:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5421 00000100:00080000:3.0:1769078433.440781:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078433.440785:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078433.440791:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078433.440804:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612807168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078433.442058:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612806784:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078433.442129:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078433.442165:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff8b10037ddc00 x1855007612806784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.442185:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612806784/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.442197:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.442202:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078433.442206:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612806784:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078433.442214:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806912:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078433.442238:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078433.442253:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff8b10037dd880 x1855007612806912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.442267:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612806912/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.442291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.442295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078433.442299:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612806912:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078433.442306:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807040:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078433.442330:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078433.442344:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff8b10037df100 x1855007612807040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.442358:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612807040/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.442367:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.442371:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078433.442376:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807040:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078433.442382:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807168:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078433.442406:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078433.442419:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff8b10037dc000 x1855007612807168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.442432:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612807168/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.442441:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.442445:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078433.442448:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807168:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078434.335788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078434.335815:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078435.359811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078435.359888:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078436.383725:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078436.383882:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078437.407861:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078437.407942:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078438.431760:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078438.431908:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078438.561272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.561287:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078438.561292:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.561305:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5426 00000100:00080000:3.0:1769078438.561320:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.561329:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078438.561376:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612807296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078438.561415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.561419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078438.561420:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.561425:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5426 00000100:00080000:3.0:1769078438.561429:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.561433:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078438.561443:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078438.561454:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612807424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078438.561462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.561465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078438.561467:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.561471:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5426 00000100:00080000:3.0:1769078438.561475:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.561479:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078438.561484:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078438.561494:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612807552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078438.561503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.561506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078438.561508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.561511:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5426 00000100:00080000:3.0:1769078438.561515:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.561518:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078438.561522:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078438.561531:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612807680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078438.561666:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612807296:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078438.561745:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078438.561785:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff8b10037df800 x1855007612807296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.561802:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612807296/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.561813:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.561818:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078438.561821:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612807296:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078438.561828:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807424:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078438.561848:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078438.561859:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff8b10037dc700 x1855007612807424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.561876:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612807424/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.561885:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.561888:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078438.561891:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807424:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078438.561896:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807552:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078438.561913:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078438.561923:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff8b10037dce00 x1855007612807552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.561934:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612807552/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.561941:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.561945:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078438.561948:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807552:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078438.561953:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807680:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078438.561968:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078438.561977:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff8b10037df480 x1855007612807680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.561988:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612807680/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.562001:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.562004:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078438.562007:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807680:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769078439.455851:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078439.455953:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078440.479804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078440.479884:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078441.503788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078441.503899:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078442.527792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078442.527931:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078443.551788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078443.551892:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078443.679785:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078443.679805:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078443.679810:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078443.679823:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5431 00000100:00080000:3.0:1769078443.679837:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078443.679846:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078443.679901:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612807808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078443.679937:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078443.679944:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078443.679947:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078443.679951:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5431 00000100:00080000:3.0:1769078443.679955:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078443.679959:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078443.679971:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078443.679984:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612807936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078443.679993:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078443.679996:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078443.679998:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078443.680002:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5431 00000100:00080000:3.0:1769078443.680005:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078443.680009:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078443.680014:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078443.680024:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612808064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078443.680032:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078443.680035:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078443.680037:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078443.680040:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5431 00000100:00080000:3.0:1769078443.680043:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078443.680046:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078443.680051:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078443.680060:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612808192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078443.681881:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612807808:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078443.681956:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078443.681996:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078443] req@ffff8b10037dea00 x1855007612807808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078443.682014:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612807808/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078443.682025:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078443.682029:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078443.682033:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612807808:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078443.682040:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807936:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078443.682058:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078443.682068:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078443] req@ffff8b10037df480 x1855007612807936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078443.682080:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612807936/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078443.682088:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078443.682091:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078443.682095:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612807936:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078443.682101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808064:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078443.682134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078443.682144:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078443] req@ffff8b10037dce00 x1855007612808064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078443.682155:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612808064/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078443.682164:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078443.682167:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078443.682170:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808064:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078443.682175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808192:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078443.682191:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078443.682199:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078443] req@ffff8b10037dc700 x1855007612808192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078443.682210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612808192/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078443.682218:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078443.682221:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078443.682224:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808192:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769078444.575854:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078444.576176:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078445.599778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078445.599880:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078446.623764:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078446.624276:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078447.647895:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078447.647920:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078448.672190:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078448.672334:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078448.800256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078448.800267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078448.800271:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078448.800280:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5436 00000100:00080000:2.0:1769078448.800296:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078448.800304:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078448.800339:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58380 x1855007612808320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078448.800366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078448.800370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078448.800372:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078448.800380:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5436 00000100:00080000:2.0:1769078448.800384:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078448.800392:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078448.800406:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078448.800421:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5ad80 x1855007612808448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078448.800433:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078448.800443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769078448.800445:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612808320:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078448.800447:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078448.800461:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5436 00000100:00080000:2.0:1769078448.800466:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078448.800469:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078448.800475:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078448.800492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b100 x1855007612808576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078448.800502:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078448.800504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078448.800506:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078448.800509:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5436 00000100:00080000:2.0:1769078448.800513:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078448.800517:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078448.800521:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078448.800542:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5aa00 x1855007612808704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078448.800553:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078448.800624:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff8b1005d58380 x1855007612808320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078448.800702:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58380 x1855007612808320/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078448.800714:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078448.800720:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078448.800724:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612808320:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078448.800757:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808448:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078448.800982:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078448.800998:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff8b1005d5ad80 x1855007612808448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078448.801012:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5ad80 x1855007612808448/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078448.801048:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078448.801052:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078448.801055:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808448:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078448.801062:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808576:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078448.801086:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078448.801097:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff8b1005d5b100 x1855007612808576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078448.801110:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b100 x1855007612808576/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078448.801149:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078448.801153:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078448.801156:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808576:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078448.801161:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808704:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078448.801182:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078448.801196:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff8b1005d5aa00 x1855007612808704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078448.801207:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5aa00 x1855007612808704/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078448.801215:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078448.801218:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078448.801221:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808704:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078449.695792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078449.699081:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078450.719788:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078450.719952:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078451.743868:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078451.744030:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078452.767899:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078452.768233:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078453.793208:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078453.793273:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078453.919800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078453.919815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078453.919819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078453.919829:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5442 00000100:00080000:3.0:1769078453.919841:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078453.919849:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078453.919891:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612808832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078453.919917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078453.919920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078453.919921:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078453.919924:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5442 00000100:00080000:3.0:1769078453.919926:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078453.919929:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078453.919955:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078453.919965:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612808960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078453.919972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078453.919974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078453.919976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078453.919978:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5442 00000100:00080000:3.0:1769078453.919981:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078453.919984:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078453.919987:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078453.919993:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612809088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078453.919998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078453.920000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078453.920001:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078453.920004:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5442 00000100:00080000:3.0:1769078453.920005:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078453.920007:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078453.920010:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078453.920014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612809216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078453.920099:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612808832:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078453.920164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078453.920198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff8b10037df800 x1855007612808832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078453.920212:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612808832/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078453.920219:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078453.920223:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078453.920225:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612808832:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078453.920231:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808960:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078453.920244:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078453.920252:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff8b10037dc000 x1855007612808960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078453.920260:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612808960/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078453.920265:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078453.920266:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078453.920268:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612808960:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078453.920271:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809088:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078453.920282:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078453.920287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff8b10037df100 x1855007612809088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078453.920294:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612809088/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078453.920299:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078453.920300:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078453.920302:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809088:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078453.920304:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809216:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078453.920314:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078453.920319:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff8b10037dd880 x1855007612809216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078453.920326:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612809216/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078453.920331:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078453.920332:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078453.920334:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809216:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078454.815765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078454.816819:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078455.839779:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078455.839797:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078456.863793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078456.863880:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078457.887778:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078457.887924:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078458.911777:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078458.911846:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078459.040055:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078459.040071:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078459.040076:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078459.040090:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5447 00000100:00080000:3.0:1769078459.040106:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078459.040115:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078459.040175:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612809344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078459.040210:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078459.040217:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078459.040219:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078459.040223:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5447 00000100:00080000:3.0:1769078459.040227:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078459.040231:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078459.040243:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078459.040256:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612809472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078459.040265:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078459.040268:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078459.040270:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078459.040274:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5447 00000100:00080000:3.0:1769078459.040277:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078459.040281:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078459.040285:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078459.040295:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612809600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078459.040302:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078459.040306:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078459.040307:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078459.040311:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5447 00000100:00080000:3.0:1769078459.040314:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078459.040318:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078459.040323:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078459.040331:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612809728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078459.040915:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612809344:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078459.040999:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078459.041036:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff8b10037ddc00 x1855007612809344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078459.041051:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612809344/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078459.041059:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078459.041062:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078459.041064:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612809344:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078459.041070:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809472:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078459.041084:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078459.041092:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff8b10037dd500 x1855007612809472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078459.041099:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612809472/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078459.041109:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078459.041111:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078459.041113:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809472:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078459.041116:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809600:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078459.041127:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078459.041132:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff8b10037ddf80 x1855007612809600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078459.041139:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612809600/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078459.041145:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078459.041148:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078459.041151:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809600:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078459.041157:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809728:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078459.041176:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078459.041187:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078459/real 1769078459] req@ffff8b10037de300 x1855007612809728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078459.041199:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612809728/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078459.041205:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078459.041207:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078459.041209:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809728:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078459.935765:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078459.935872:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078460.959878:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078460.961060:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078461.983793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078461.983840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078463.007775:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078463.007856:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078464.031793:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078464.031898:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078464.159742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.159753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078464.159775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.159787:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5452 00000100:00080000:3.0:1769078464.159800:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.159809:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078464.159859:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612809856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078464.159888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.159891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078464.159892:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.159896:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5452 00000100:00080000:3.0:1769078464.159898:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.159901:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078464.159910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078464.159919:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612809984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078464.159925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.159928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078464.159929:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.159931:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5452 00000100:00080000:3.0:1769078464.159934:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.159937:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078464.159939:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078464.159948:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612810112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078464.159955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.159958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078464.159960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.159963:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5452 00000100:00080000:3.0:1769078464.159967:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.159970:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078464.159975:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078464.159985:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612810240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078464.162871:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612809856:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078464.162972:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078464.163016:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff8b10037ded80 x1855007612809856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078464.163036:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612809856/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078464.163047:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078464.163051:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078464.163055:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612809856:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078464.163063:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809984:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078464.163083:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078464.163095:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff8b10037dca80 x1855007612809984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078464.163106:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612809984/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078464.163113:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078464.163116:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078464.163120:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612809984:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078464.163125:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810112:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078464.163143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078464.163153:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff8b10037dc380 x1855007612810112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078464.163164:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612810112/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078464.163172:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078464.163175:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078464.163178:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810112:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078464.163184:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810240:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078464.163200:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078464.163209:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff8b10037dd180 x1855007612810240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078464.163220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612810240/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078464.163228:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078464.163231:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078464.163234:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810240:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078465.056046:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078465.056361:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078466.079782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078466.081507:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078467.103874:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078467.104000:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078468.130483:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078468.130589:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078469.151854:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078469.152851:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078469.279850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078469.279861:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078469.279866:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078469.279878:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5457 00000100:00080000:3.0:1769078469.279891:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078469.279899:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078469.279938:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de680 x1855007612810368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078469.279977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078469.279983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078469.279985:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078469.279989:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5457 00000100:00080000:3.0:1769078469.279992:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078469.279997:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078469.280006:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078469.280022:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612810496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078469.280032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078469.280036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078469.280038:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078469.280042:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5457 00000100:00080000:3.0:1769078469.280051:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078469.280055:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078469.280061:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078469.280070:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612810368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078469.280073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612810624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078469.280087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078469.280090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078469.280092:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078469.280096:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5457 00000100:00080000:3.0:1769078469.280099:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078469.280103:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078469.280109:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078469.280119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612810752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078469.280164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078469.280200:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff8b10037de680 x1855007612810368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.280220:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de680 x1855007612810368/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.280230:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.280236:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078469.280241:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612810368:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078469.280249:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810496:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078469.280270:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078469.280279:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff8b10037dd180 x1855007612810496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.280291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612810496/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.280296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.280299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078469.280301:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810496:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078469.280323:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810624:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078469.280351:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078469.280358:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff8b10037dc380 x1855007612810624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.280366:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612810624/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.280371:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.280373:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078469.280375:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810624:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078469.280378:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810752:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078469.280388:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078469.280394:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff8b10037dca80 x1855007612810752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.280401:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612810752/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.280405:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.280407:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078469.280409:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612810752:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078470.175780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078470.182722:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078471.199787:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078471.200032:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078472.223757:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078472.223809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078473.247717:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078473.247775:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078474.271746:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078474.271782:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078474.399770:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078474.399781:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078474.399792:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078474.399803:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5462 00000100:00080000:3.0:1769078474.399815:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078474.399822:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078474.399866:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612810880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078474.399892:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078474.399895:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078474.399896:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078474.399900:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5462 00000100:00080000:3.0:1769078474.399902:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078474.399904:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078474.399913:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078474.399924:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612811008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078474.399931:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078474.399934:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078474.399935:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078474.399937:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5462 00000100:00080000:3.0:1769078474.399941:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078474.399945:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078474.399953:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078474.399954:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612810880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:3.0:1769078474.399965:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612811136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078474.399974:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078474.399976:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078474.399977:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078474.399979:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5462 00000100:00080000:3.0:1769078474.399981:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078474.399984:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078474.399989:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078474.399997:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612811264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078474.400028:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078474.400060:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff8b10037ded80 x1855007612810880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.400073:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612810880/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.400081:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.400084:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078474.400087:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612810880:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078474.400092:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811008:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078474.400105:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078474.400112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff8b10037de300 x1855007612811008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.400120:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612811008/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.400125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.400127:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078474.400128:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811008:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078474.400148:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811136:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078474.400159:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078474.400164:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff8b10037ddf80 x1855007612811136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.400171:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612811136/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.400176:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.400178:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078474.400179:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811136:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078474.400183:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811264:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078474.400193:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078474.400198:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff8b10037dd500 x1855007612811264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.400205:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612811264/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.400209:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.400211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078474.400213:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811264:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078475.295729:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078475.295777:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078476.319731:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078476.319800:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078477.343751:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078477.343813:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078478.367755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078478.367819:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078479.391871:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078479.391996:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078479.519800:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078479.519817:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078479.519823:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078479.519837:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5467 00000100:00080000:3.0:1769078479.519853:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078479.519862:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078479.519934:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612811392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078479.519975:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078479.519983:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078479.519985:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078479.519990:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5467 00000100:00080000:3.0:1769078479.519995:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078479.519999:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078479.520012:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078479.520027:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612811520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078479.520041:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612811392:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078479.520042:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078479.520053:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078479.520055:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078479.520061:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5467 00000100:00080000:3.0:1769078479.520066:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078479.520070:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078479.520076:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078479.520093:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612811648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078479.520104:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078479.520108:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078479.520110:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078479.520113:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5467 00000100:00080000:3.0:1769078479.520118:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078479.520122:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078479.520128:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078479.520138:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078479.520141:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612811776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078479.520183:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff8b10037ddc00 x1855007612811392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.520219:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612811392/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.520233:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.520238:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078479.520244:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612811392:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078479.520275:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811520:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078479.520301:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078479.520316:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff8b10037dd880 x1855007612811520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.520327:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612811520/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.520333:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.520336:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078479.520338:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811520:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078479.520343:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811648:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078479.520359:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078479.520367:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff8b10037df100 x1855007612811648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.520382:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612811648/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.520390:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.520394:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078479.520397:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811648:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078479.520403:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811776:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078479.520421:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078479.520432:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff8b10037dc000 x1855007612811776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.520445:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612811776/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.520454:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.520457:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078479.520460:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612811776:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078480.415762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078480.415830:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078481.439786:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078481.439843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078482.463795:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078482.464608:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078483.488172:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078483.488172:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078484.512473:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078484.512679:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078484.639793:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.639806:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078484.639812:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.639823:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5472 00000100:00080000:3.0:1769078484.639838:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.639847:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078484.639894:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612811904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.639923:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.639926:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078484.639927:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.639930:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5472 00000100:00080000:3.0:1769078484.639933:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.639936:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078484.639944:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078484.639953:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612812032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.639959:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.639962:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078484.639963:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.639965:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5472 00000100:00080000:3.0:1769078484.639968:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.639970:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078484.639973:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078484.639978:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612812160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.639984:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.639986:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078484.639987:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.639989:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5472 00000100:00080000:3.0:1769078484.639990:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.639992:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078484.639995:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078484.640000:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612812288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078484.640149:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612811904:192.168.204.102@tcp:250:kworker.0 00000800:00000100:0.0:1769078484.640229:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078484.640273:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff8b10037df800 x1855007612811904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.640291:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612811904/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.640298:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.640302:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078484.640306:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612811904:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078484.640313:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812032:192.168.204.102@tcp:38:kworker.0 00000800:00000100:0.0:1769078484.640331:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078484.640343:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff8b10037dc700 x1855007612812032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.640355:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612812032/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.640361:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.640363:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078484.640366:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812032:192.168.204.102@tcp:38:kworker.0 00000100:00100000:0.0:1769078484.640372:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812160:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078484.640390:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078484.640400:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff8b10037dce00 x1855007612812160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.640408:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612812160/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.640430:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.640434:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078484.640437:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812160:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078484.640441:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812288:192.168.204.102@tcp:8:kworker.0 00000800:00000100:0.0:1769078484.640456:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078484.640466:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff8b10037df480 x1855007612812288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.640475:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612812288/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.640480:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.640482:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:0.0:1769078484.640484:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812288:192.168.204.102@tcp:8:kworker.0 00000100:00100000:0.0:1769078485.535853:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078485.535998:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078486.559818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078486.562156:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078487.583818:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078487.584531:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078488.607791:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078488.608572:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078489.631847:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078489.633061:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078489.760585:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078489.760598:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078489.760602:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078489.760612:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5477 00000100:00080000:0.0:1769078489.760622:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078489.760628:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078489.760687:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025880 x1855007612812416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078489.760729:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078489.760733:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078489.760735:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078489.760739:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5477 00000100:00080000:0.0:1769078489.760743:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078489.760748:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078489.760757:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078489.760768:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024000 x1855007612812544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078489.760778:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078489.760782:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078489.760783:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078489.760787:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5477 00000100:00080000:0.0:1769078489.760790:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078489.760793:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078489.760799:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078489.760809:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026a00 x1855007612812672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078489.760818:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078489.760821:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078489.760822:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078489.760826:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5477 00000100:00080000:0.0:1769078489.760829:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078489.760832:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078489.760837:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078489.760845:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026300 x1855007612812800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078489.760931:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612812416:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078489.760994:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078489.761033:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff8b1006025880 x1855007612812416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078489.761052:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025880 x1855007612812416/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078489.761062:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078489.761067:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078489.761070:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612812416:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078489.761077:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812544:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078489.761099:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078489.761112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff8b1006024000 x1855007612812544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078489.761125:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024000 x1855007612812544/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078489.761132:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078489.761136:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078489.761139:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812544:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078489.761145:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812672:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078489.761164:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078489.761175:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff8b1006026a00 x1855007612812672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078489.761186:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026a00 x1855007612812672/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078489.761194:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078489.761197:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078489.761200:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812672:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078489.761206:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812800:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078489.761224:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078489.761234:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff8b1006026300 x1855007612812800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078489.761246:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026300 x1855007612812800/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078489.761253:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078489.761256:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078489.761259:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612812800:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769078490.655866:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078490.656078:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078491.680086:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078491.680762:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078492.703852:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078492.704357:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078493.727930:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078493.728569:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078494.751831:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078494.753374:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078494.879783:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.879797:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078494.879803:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.879815:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5482 00000100:00080000:3.0:1769078494.879830:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.879838:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078494.879887:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dea00 x1855007612812928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.879925:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.879931:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078494.879934:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.879938:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5482 00000100:00080000:3.0:1769078494.879942:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.879947:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078494.879958:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078494.879973:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df480 x1855007612813056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.879983:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.879986:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078494.879988:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.879992:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5482 00000100:00080000:3.0:1769078494.879996:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.880000:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078494.880005:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078494.880016:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dce00 x1855007612813184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.880025:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.880029:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078494.880030:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.880034:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5482 00000100:00080000:3.0:1769078494.880037:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.880041:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078494.880046:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078494.880056:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc700 x1855007612813312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078494.881021:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612812928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078494.881129:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078494.881155:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078494.881175:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813312:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769078495.776031:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078495.776180:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078496.799815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078496.799843:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078497.824061:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078497.824095:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000400:00000100:0.0:1769078497.952027:0:2354:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.102:988 00000400:00000100:0.0:1769078497.952074:0:2354:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.102@tcp at host 192.168.204.102 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078497.952103:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769078497.952135:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769078497.952143:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000800:00000100:0.0:1769078497.952151:0:2354:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.102@tcp 00000100:00100000:1.0:1769078497.952175:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078497] req@ffff8b10037dea00 x1855007612812928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078497.952210:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dea00 x1855007612812928/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078497.952222:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078497.952227:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078497.952232:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612812928:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078497.952240:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078497] req@ffff8b10037df480 x1855007612813056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078497.952252:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df480 x1855007612813056/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078497.952259:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078497.952262:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078497.952266:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813056:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078497.952272:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078497] req@ffff8b10037dce00 x1855007612813184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078497.952284:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dce00 x1855007612813184/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078497.952291:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078497.952295:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078497.952298:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813184:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078497.952304:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078497] req@ffff8b10037dc700 x1855007612813312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078497.952321:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc700 x1855007612813312/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078497.952330:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078497.952333:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078497.952337:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813312:192.168.204.102@tcp:8:kworker.0 00000100:00100000:2.0:1769078498.847724:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078498.848657:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078499.871832:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078499.871887:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:2.0:1769078499.999944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.999958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078499.999964:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.999977:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5488 00000100:00080000:2.0:1769078499.999991:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078500.000000:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078500.001976:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59880 x1855007612813440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078500.002027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078500.002033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078500.002037:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078500.002043:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5488 00000100:00080000:2.0:1769078500.002057:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078500.002066:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078500.002086:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078500.002132:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5bb80 x1855007612813568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078500.002143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078500.002147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078500.002149:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078500.002166:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5488 00000100:00080000:2.0:1769078500.002171:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078500.002176:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078500.002184:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078500.002196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b480 x1855007612813696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078500.002205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078500.002208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078500.002211:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078500.002215:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5488 00000100:00080000:2.0:1769078500.002218:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078500.002222:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078500.002227:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078500.002243:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59500 x1855007612813824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078500.002694:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612813440:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078500.005589:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078500.005664:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff8b1005d59880 x1855007612813440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078500.005690:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59880 x1855007612813440/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078500.005703:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078500.005709:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078500.005713:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612813440:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078500.005727:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813568:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078500.005760:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078500.005774:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff8b1005d5bb80 x1855007612813568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078500.005787:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5bb80 x1855007612813568/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078500.005795:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078500.005799:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078500.005802:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813568:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078500.005809:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813696:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078500.005847:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078500.005859:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff8b1005d5b480 x1855007612813696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078500.005871:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b480 x1855007612813696/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078500.005879:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078500.005883:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078500.005887:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813696:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078500.005894:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813824:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078500.005914:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078500.005926:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078500/real 1769078500] req@ffff8b1005d59500 x1855007612813824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078500.005938:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59500 x1855007612813824/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078500.005945:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078500.005949:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078500.005952:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612813824:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078500.895790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078500.895935:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078501.923373:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078501.923441:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078502.944380:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078502.944596:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078503.967804:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078503.968039:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078504.993909:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078504.994310:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078505.119928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078505.119941:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078505.119946:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078505.119957:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5493 00000100:00080000:2.0:1769078505.119970:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078505.119978:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078505.120023:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5b800 x1855007612813952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078505.120066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078505.120095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078505.120098:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078505.120103:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5493 00000100:00080000:2.0:1769078505.120107:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078505.120112:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078505.120124:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078505.120138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58a80 x1855007612814080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078505.120140:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612813952:192.168.204.102@tcp:250:kworker.0 00000100:00080000:2.0:1769078505.120155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078505.120158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078505.120160:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078505.120165:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5493 00000100:00080000:2.0:1769078505.120169:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078505.120172:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078505.120177:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078505.120188:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612814208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078505.120197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078505.120200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078505.120202:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078505.120206:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5493 00000100:00080000:2.0:1769078505.120210:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078505.120213:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078505.120219:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078505.120228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612814336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078505.120233:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078505.120286:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff8b1005d5b800 x1855007612813952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078505.120317:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5b800 x1855007612813952/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078505.120330:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078505.120335:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078505.120341:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612813952:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078505.120371:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814080:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078505.120396:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078505.120411:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff8b1005d58a80 x1855007612814080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078505.120424:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58a80 x1855007612814080/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078505.120432:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078505.120436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078505.120440:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814080:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078505.120447:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814208:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078505.120472:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078505.120486:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff8b1005d58e00 x1855007612814208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078505.120498:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612814208/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078505.120512:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078505.120516:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078505.120520:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814208:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078505.120525:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814336:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078505.120547:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078505.120558:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff8b1005d5a300 x1855007612814336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078505.120570:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612814336/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078505.120578:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078505.120581:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078505.120584:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814336:192.168.204.102@tcp:8:kworker.0 00010000:00010000:2.0:1769078506.016074:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078506.016611:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078507.039811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078507.039872:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078508.063780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078508.063869:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078509.087754:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078509.087803:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078510.111752:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078510.111827:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078510.239730:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078510.239740:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078510.239745:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078510.239758:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5498 00000100:00080000:0.0:1769078510.239770:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078510.239776:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078510.239814:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006026d80 x1855007612814464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078510.239835:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078510.239838:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078510.239839:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078510.239842:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5498 00000100:00080000:0.0:1769078510.239844:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078510.239846:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078510.239855:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078510.239865:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027b80 x1855007612814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078510.239872:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078510.239876:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078510.239878:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769078510.239879:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612814464:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078510.239881:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5498 00000100:00080000:0.0:1769078510.239886:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078510.239889:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078510.239892:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078510.239900:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006025f80 x1855007612814720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078510.239910:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078510.239913:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078510.239914:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078510.239916:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5498 00000100:00080000:0.0:1769078510.239918:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078510.239920:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078510.239923:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078510.239929:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024a80 x1855007612814848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078510.239953:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078510.240004:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078510] req@ffff8b1006026d80 x1855007612814464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078510.240031:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006026d80 x1855007612814464/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078510.240040:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078510.240043:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078510.240046:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006026d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612814464:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078510.240073:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814592:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078510.240098:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078510.240112:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078510] req@ffff8b1006027b80 x1855007612814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078510.240124:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027b80 x1855007612814592/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078510.240130:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078510.240132:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078510.240134:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814592:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078510.240139:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814720:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078510.240157:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078510.240169:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078510] req@ffff8b1006025f80 x1855007612814720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078510.240181:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006025f80 x1855007612814720/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078510.240188:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078510.240192:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078510.240195:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006025f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814720:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078510.240201:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814848:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078510.240220:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078510.240228:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078510] req@ffff8b1006024a80 x1855007612814848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078510.240236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024a80 x1855007612814848/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078510.240242:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078510.240244:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078510.240246:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612814848:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078511.135744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078511.135803:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078512.159744:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078512.159838:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078513.183768:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078513.183842:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078514.207761:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078514.207840:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078515.231755:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078515.231809:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078515.359742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078515.359750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078515.359753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078515.359760:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5503 00000100:00080000:0.0:1769078515.359769:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078515.359773:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078515.359803:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612814976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078515.359825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078515.359828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078515.359830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078515.359836:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5503 00000100:00080000:0.0:1769078515.359839:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078515.359842:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078515.359851:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078515.359862:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612815104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078515.359875:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612814976:192.168.204.102@tcp:250:kworker.0 00000100:00080000:0.0:1769078515.359879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078515.359882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078515.359883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078515.359886:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5503 00000100:00080000:0.0:1769078515.359889:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078515.359893:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078515.359910:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078515.359920:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612815232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078515.359927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078515.359930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078515.359931:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078515.359934:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5503 00000100:00080000:0.0:1769078515.359936:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078515.359939:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769078515.359941:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:0.0:1769078515.359943:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078515.359951:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612815360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078515.359971:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff8b1006024700 x1855007612814976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078515.359986:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612814976/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078515.359995:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078515.359999:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078515.360003:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612814976:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078515.360021:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815104:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078515.360038:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078515.360046:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff8b1006024e00 x1855007612815104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078515.360053:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612815104/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078515.360062:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078515.360064:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078515.360065:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815104:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078515.360069:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815232:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078515.360078:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078515.360083:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff8b1006024380 x1855007612815232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078515.360090:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612815232/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078515.360095:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078515.360096:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078515.360098:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815232:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078515.360101:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815360:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078515.360109:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078515.360115:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff8b1006027480 x1855007612815360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078515.360121:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612815360/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078515.360125:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078515.360126:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078515.360129:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815360:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078516.256008:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078516.256341:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078517.279781:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078517.279973:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078518.303792:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078518.303877:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078519.327750:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078519.327810:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078520.351782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078520.351838:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078520.479849:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078520.479865:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078520.479871:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078520.479884:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5508 00000100:00080000:3.0:1769078520.479898:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078520.479907:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078520.479956:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df800 x1855007612815488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078520.479986:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078520.479990:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078520.479992:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078520.479997:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5508 00000100:00080000:3.0:1769078520.480001:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078520.480006:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078520.480019:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078520.480034:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc000 x1855007612815616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078520.480040:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612815488:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078520.480050:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078520.480055:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078520.480058:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078520.480063:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5508 00000100:00080000:3.0:1769078520.480068:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078520.480072:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078520.480077:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078520.480093:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037df100 x1855007612815744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078520.480103:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078520.480107:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078520.480108:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078520.480112:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5508 00000100:00080000:3.0:1769078520.480117:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078520.480121:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000800:00000100:1.0:1769078520.480126:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000008:00100000:3.0:1769078520.480127:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078520.480142:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd880 x1855007612815872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078520.480175:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff8b10037df800 x1855007612815488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078520.480195:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df800 x1855007612815488/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078520.480206:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078520.480211:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078520.480215:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612815488:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078520.480243:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815616:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078520.480264:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078520.480275:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff8b10037dc000 x1855007612815616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078520.480288:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc000 x1855007612815616/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078520.480296:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078520.480299:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078520.480303:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815616:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078520.480309:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815744:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078520.480327:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078520.480347:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff8b10037df100 x1855007612815744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078520.480359:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037df100 x1855007612815744/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078520.480367:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078520.480370:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078520.480373:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815744:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078520.480378:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815872:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078520.480404:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078520.480415:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff8b10037dd880 x1855007612815872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078520.480426:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd880 x1855007612815872/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078520.480433:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078520.480436:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078520.480439:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612815872:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078521.375782:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078521.377846:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078522.399818:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078522.402904:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078523.423811:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078523.424022:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078524.447950:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078524.447980:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078525.471823:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078525.471944:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078525.599879:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078525.599894:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078525.599899:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078525.599911:0:37199:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5513 00000100:00080000:3.0:1769078525.599926:0:37199:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078525.599934:0:37199:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078525.599982:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddc00 x1855007612816000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078525.600021:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078525.600025:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078525.600027:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078525.600031:0:37199:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5513 00000100:00080000:3.0:1769078525.600036:0:37199:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078525.600041:0:37199:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078525.600052:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078525.600066:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd500 x1855007612816128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078525.600076:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078525.600079:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078525.600081:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078525.600085:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5513 00000100:00080000:3.0:1769078525.600088:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078525.600092:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078525.600097:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078525.600119:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ddf80 x1855007612816256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078525.600129:0:37199:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078525.600132:0:37199:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078525.600134:0:37199:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078525.600138:0:37199:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5513 00000100:00080000:3.0:1769078525.600141:0:37199:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078525.600146:0:37199:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078525.600151:0:37199:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078525.600162:0:37199:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037de300 x1855007612816384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078525.600798:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612816000:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078525.600880:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078525.600923:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff8b10037ddc00 x1855007612816000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078525.600953:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddc00 x1855007612816000/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078525.600965:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078525.600970:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078525.600974:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612816000:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078525.600982:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816128:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078525.601002:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078525.601016:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff8b10037dd500 x1855007612816128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078525.601028:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd500 x1855007612816128/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078525.601037:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078525.601040:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078525.601044:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816128:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078525.601050:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816256:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078525.601068:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078525.601078:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff8b10037ddf80 x1855007612816256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078525.601089:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ddf80 x1855007612816256/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078525.601096:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078525.601099:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078525.601102:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816256:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078525.601107:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816384:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078525.601123:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078525.601133:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff8b10037de300 x1855007612816384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078525.601143:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037de300 x1855007612816384/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078525.601151:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078525.601154:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078525.601157:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816384:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078526.495899:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078526.496050:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078527.519815:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078527.519869:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078528.543780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078528.544778:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769078529.568295:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078529.568426:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078530.591790:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078530.592012:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078530.721088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078530.721113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078530.721119:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078530.721131:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5518 00000100:00080000:0.0:1769078530.721146:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078530.721163:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:0.0:1769078530.721783:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006027480 x1855007612816512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078530.721854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078530.721858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078530.721860:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078530.721865:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5518 00000100:00080000:0.0:1769078530.721870:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078530.721875:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078530.721887:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078530.721906:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612816512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:0.0:1769078530.721912:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024380 x1855007612816640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078530.721925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078530.721929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078530.721931:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078530.721936:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5518 00000100:00080000:0.0:1769078530.721939:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078530.721943:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078530.721948:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078530.721961:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024e00 x1855007612816768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078530.721969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078530.721973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078530.721976:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078530.721980:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5518 00000100:00080000:0.0:1769078530.721983:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078530.721987:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:0.0:1769078530.721992:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078530.721993:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:0.0:1769078530.722007:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1006024700 x1855007612816896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078530.722045:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff8b1006027480 x1855007612816512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078530.722067:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006027480 x1855007612816512/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078530.722079:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078530.722084:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078530.722089:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006027480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612816512:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078530.722119:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816640:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078530.722143:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078530.722156:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff8b1006024380 x1855007612816640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078530.722170:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024380 x1855007612816640/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078530.722178:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078530.722182:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078530.722185:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816640:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078530.722191:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816768:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078530.722212:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078530.722224:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff8b1006024e00 x1855007612816768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078530.722236:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024e00 x1855007612816768/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078530.722244:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078530.722248:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078530.722251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816768:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078530.722257:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816896:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078530.722276:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078530.722287:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff8b1006024700 x1855007612816896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078530.722298:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1006024700 x1855007612816896/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078530.722312:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078530.722316:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078530.722319:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1006024700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612816896:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078531.615809:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078531.615951:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078532.639780:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078532.639826:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078533.663950:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078533.664003:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078534.687889:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078534.687971:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078535.712139:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078535.713947:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078535.839979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078535.839989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078535.839993:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078535.840003:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5523 00000100:00080000:2.0:1769078535.840014:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078535.840020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:2.0:1769078535.840056:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58000 x1855007612817024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078535.840090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078535.840094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078535.840096:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078535.840100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5523 00000100:00080000:2.0:1769078535.840103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078535.840107:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078535.840116:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078535.840127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d59180 x1855007612817152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078535.840137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078535.840140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078535.840142:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078535.840145:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5523 00000100:00080000:2.0:1769078535.840148:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078535.840151:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078535.840155:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078535.840165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d5a300 x1855007612817280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078535.840174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078535.840177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078535.840179:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078535.840182:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5523 00000100:00080000:2.0:1769078535.840185:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078535.840189:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:2.0:1769078535.840193:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078535.840202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1005d58e00 x1855007612817408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078535.840300:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612817024:192.168.204.102@tcp:250:kworker.0 00000800:00000100:1.0:1769078535.840355:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078535.840987:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff8b1005d58000 x1855007612817024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078535.841009:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58000 x1855007612817024/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078535.841020:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078535.841025:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078535.841029:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612817024:192.168.204.102@tcp:250:kworker.0 00000100:00100000:1.0:1769078535.841038:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817152:192.168.204.102@tcp:38:kworker.0 00000800:00000100:1.0:1769078535.841066:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078535.841080:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff8b1005d59180 x1855007612817152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078535.841093:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d59180 x1855007612817152/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078535.841101:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078535.841104:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078535.841108:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d59180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817152:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078535.841115:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817280:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078535.841134:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078535.841145:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff8b1005d5a300 x1855007612817280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078535.841157:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d5a300 x1855007612817280/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078535.841165:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078535.841168:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078535.841171:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d5a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817280:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078535.841177:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817408:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078535.841195:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078535.841205:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff8b1005d58e00 x1855007612817408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078535.841217:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1005d58e00 x1855007612817408/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078535.841224:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078535.841228:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078535.841231:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b1005d58e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817408:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078536.735770:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078536.735852:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078537.759767:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078537.760615:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078538.783735:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078538.783795:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078539.807699:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078539.807746:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769078540.831841:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078540.831890:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078540.959777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 86724556-597d-4d0e-ac75-6d9d64145050->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078540.959789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078540.959793:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078540.959802:0:37:0:(import.c:509:import_select_connection()) MGC192.168.204.102@tcp: connect to NID 192.168.204.102@tcp last attempt 5529 00000100:00080000:3.0:1769078540.959813:0:37:0:(import.c:588:import_select_connection()) MGC192.168.204.102@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078540.959819:0:37:0:(import.c:623:import_select_connection()) MGC192.168.204.102@tcp: import ffff8b10096a0800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000100:00100000:3.0:1769078540.959852:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037ded80 x1855007612817536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078540.959882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078540.959885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078540.959888:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769078540.959932:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612817536:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078540.959964:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5529 00000100:00080000:3.0:1769078540.959969:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078540.959973:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff8b101127b800: import ffff8b1008679800 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078540.959981:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769078540.959991:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:3.0:1769078540.959997:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dca80 x1855007612817664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078540.960012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078540.960015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078540.960017:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078540.960022:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5529 00000100:00100000:1.0:1769078540.960027:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff8b10037ded80 x1855007612817536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.960028:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078540.960036:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff8b101127b800: import ffff8b1011207000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078540.960045:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078540.960053:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037ded80 x1855007612817536/t0(0) o250->MGC192.168.204.102@tcp@192.168.204.102@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.960059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dc380 x1855007612817792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078540.960066:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b10096a0800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078540.960074:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.102@tcp failed (-110) 00000100:00080000:3.0:1769078540.960074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 93598d0a-9484-4e40-94b2-393ee542cbad->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078540.960082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769078540.960083:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037ded80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:86724556-597d-4d0e-ac75-6d9d64145050:2363:1855007612817536:192.168.204.102@tcp:250:kworker.0 00000100:00080000:3.0:1769078540.960086:0:37:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078540.960094:0:37:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: connect to NID 192.168.204.102@tcp last attempt 5529 00000100:00080000:3.0:1769078540.960098:0:37:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078540.960102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff8b101127b800: import ffff8b1008678000 using connection 192.168.204.102@tcp/192.168.204.102@tcp 00000008:00100000:3.0:1769078540.960107:0:37:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078540.960112:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:3.0:1769078540.960121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b10037dd180 x1855007612817920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078540.960140:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078540.960154:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff8b10037dca80 x1855007612817664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078540.960167:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dca80 x1855007612817664/t0(0) o38->lustre-MDT0000-mdc-ffff8b101127b800@192.168.204.102@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078540.960174:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008679800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078540.960177:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078540.960181:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817664:192.168.204.102@tcp:38:kworker.0 00000100:00100000:1.0:1769078540.960188:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817792:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078540.960206:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078540.960215:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff8b10037dc380 x1855007612817792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078540.960225:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dc380 x1855007612817792/t0(0) o8->lustre-OST0000-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078540.960245:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1011207000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078540.960249:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078540.960251:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817792:192.168.204.102@tcp:8:kworker.0 00000100:00100000:1.0:1769078540.960263:0:2363:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817920:192.168.204.102@tcp:8:kworker.0 00000800:00000100:1.0:1769078540.960288:0:2363:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.102@tcp 00000100:00100000:1.0:1769078540.960298:0:2363:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff8b10037dd180 x1855007612817920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078540.960309:0:2363:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b10037dd180 x1855007612817920/t0(0) o8->lustre-OST0001-osc-ffff8b101127b800@192.168.204.102@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078540.960317:0:2363:0:(import.c:65:import_set_state_nolock()) ffff8b1008678000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078540.960320:0:2363:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.102@tcp failed (-110) 00000100:00100000:1.0:1769078540.960323:0:2363:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff8b10037dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:93598d0a-9484-4e40-94b2-393ee542cbad:2363:1855007612817920:192.168.204.102@tcp:8:kworker.0 00000100:00100000:3.0:1769078541.855742:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078541.855872:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078542.879733:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078542.879900:0:29100:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078543.903905:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078543.904466:0:13997:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078544.927753:0:13998:0:(client.c:2690:ptlrpc_set_wait()) set ffff8b100364c180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078544.927853:0:29101:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 77073 lines, 77073 kept, 0 dropped, 0 bad.